all

Securite/Linkwarden/scripts/migration/migrateToV2.js

@@ -0,0 +1,122 @@
+const { S3 } = require("@aws-sdk/client-s3");
+const { PrismaClient } = require("@prisma/client");
+const { existsSync } = require("fs");
+const util = require("util");
+
+const prisma = new PrismaClient();
+
+const STORAGE_FOLDER = process.env.STORAGE_FOLDER || "data";
+
+const s3Client =
+  process.env.SPACES_ENDPOINT &&
+  process.env.SPACES_REGION &&
+  process.env.SPACES_KEY &&
+  process.env.SPACES_SECRET
+    ? new S3({
+        forcePathStyle: false,
+        endpoint: process.env.SPACES_ENDPOINT,
+        region: process.env.SPACES_REGION,
+        credentials: {
+          accessKeyId: process.env.SPACES_KEY,
+          secretAccessKey: process.env.SPACES_SECRET,
+        },
+      })
+    : undefined;
+
+async function checkFileExistence(path) {
+  if (s3Client) {
+    const bucketParams = {
+      Bucket: process.env.SPACES_BUCKET_NAME,
+      Key: path,
+    };
+
+    try {
+      const headObjectAsync = util.promisify(
+        s3Client.headObject.bind(s3Client)
+      );
+
+      try {
+        await headObjectAsync(bucketParams);
+        return true;
+      } catch (err) {
+        return false;
+      }
+    } catch (err) {
+      console.log("Error:", err);
+
+      return false;
+    }
+  } else {
+    try {
+      if (existsSync(STORAGE_FOLDER + "/" + path)) {
+        return true;
+      } else return false;
+    } catch (err) {
+      console.log(err);
+    }
+  }
+}
+
+// Avatars
+async function migrateToV2() {
+  const users = await prisma.user.findMany();
+
+  for (let user of users) {
+    const path = `uploads/avatar/${user.id}.jpg`;
+
+    const res = await checkFileExistence(path);
+
+    if (res) {
+      await prisma.user.update({
+        where: { id: user.id },
+        data: { image: path },
+      });
+      console.log(`${user.id}`);
+    } else {
+      console.log(`${user.id}`);
+    }
+  }
+
+  const links = await prisma.link.findMany();
+
+  // PDFs
+  for (let link of links) {
+    const path = `archives/${link.collectionId}/${link.id}.pdf`;
+
+    const res = await checkFileExistence(path);
+
+    if (res) {
+      await prisma.link.update({
+        where: { id: link.id },
+        data: { pdf: path },
+      });
+      console.log(`${link.id}`);
+    } else {
+      console.log(`${link.id}`);
+    }
+  }
+
+  // Screenshots
+  for (let link of links) {
+    const path = `archives/${link.collectionId}/${link.id}.png`;
+
+    const res = await checkFileExistence(path);
+
+    if (res) {
+      await prisma.link.update({
+        where: { id: link.id },
+        data: { image: path },
+      });
+      console.log(`${link.id}`);
+    } else {
+      console.log(`${link.id}`);
+    }
+  }
+
+  await prisma.$disconnect();
+}
+
+migrateToV2().catch((e) => {
+  console.error(e);
+  process.exit(1);
+});

Securite/Linkwarden/scripts/worker.ts

@@ -0,0 +1,143 @@
+import "dotenv/config";
+import { Collection, Link, User } from "@prisma/client";
+import { prisma } from "../lib/api/db";
+import archiveHandler from "../lib/api/archiveHandler";
+
+const args = process.argv.slice(2).join(" ");
+
+const archiveTakeCount = Number(process.env.ARCHIVE_TAKE_COUNT || "") || 5;
+
+type LinksAndCollectionAndOwner = Link & {
+  collection: Collection & {
+    owner: User;
+  };
+};
+
+async function processBatch() {
+  const linksOldToNew = await prisma.link.findMany({
+    where: {
+      url: { not: null },
+      OR: [
+        {
+          image: null,
+        },
+        {
+          image: "pending",
+        },
+        ///////////////////////
+        {
+          pdf: null,
+        },
+        {
+          pdf: "pending",
+        },
+        ///////////////////////
+        {
+          readable: null,
+        },
+        {
+          readable: "pending",
+        },
+      ],
+    },
+    take: archiveTakeCount,
+    orderBy: { id: "asc" },
+    include: {
+      collection: {
+        include: {
+          owner: true,
+        },
+      },
+    },
+  });
+
+  const linksNewToOld = await prisma.link.findMany({
+    where: {
+      url: { not: null },
+      OR: [
+        {
+          image: null,
+        },
+        {
+          image: "pending",
+        },
+        ///////////////////////
+        {
+          pdf: null,
+        },
+        {
+          pdf: "pending",
+        },
+        ///////////////////////
+        {
+          readable: null,
+        },
+        {
+          readable: "pending",
+        },
+      ],
+    },
+    take: archiveTakeCount,
+    orderBy: { id: "desc" },
+    include: {
+      collection: {
+        include: {
+          owner: true,
+        },
+      },
+    },
+  });
+
+  const archiveLink = async (link: LinksAndCollectionAndOwner) => {
+    try {
+      console.log(
+        "\x1b[34m%s\x1b[0m",
+        `Processing link ${link.url} for user ${link.collection.ownerId}`
+      );
+
+      await archiveHandler(link);
+
+      console.log(
+        "\x1b[34m%s\x1b[0m",
+        `Succeeded processing link ${link.url} for user ${link.collection.ownerId}.`
+      );
+    } catch (error) {
+      console.error(
+        "\x1b[34m%s\x1b[0m",
+        `Error processing link ${link.url} for user ${link.collection.ownerId}:`,
+        error
+      );
+    }
+  };
+
+  // Process each link in the batch concurrently
+  const processingPromises = [...linksOldToNew, ...linksNewToOld]
+    // Make sure we don't process the same link twice
+    .filter((value, index, self) => {
+      return self.findIndex((item) => item.id === value.id) === index;
+    })
+    .map((e) => archiveLink(e));
+
+  await Promise.allSettled(processingPromises);
+}
+
+const intervalInSeconds = Number(process.env.ARCHIVE_SCRIPT_INTERVAL) || 10;
+
+function delay(sec: number) {
+  return new Promise((resolve) => setTimeout(resolve, sec * 1000));
+}
+
+async function init() {
+  console.log("\x1b[34m%s\x1b[0m", "Starting the link processing task");
+  while (true) {
+    try {
+      await processBatch();
+      await delay(intervalInSeconds);
+    } catch (error) {
+      console.error("\x1b[34m%s\x1b[0m", "Error processing links:", error);
+      await delay(intervalInSeconds);
+    }
+  }
+}
+
+init();