feat: PostgreSQL-backed dataset search with filters, sync pipeline, and file download

backend/config/config.js

@@ -1,10 +1,19 @@
 require("dotenv").config();
 
 module.exports = {
+  // development: {
+  //   dialect: "sqlite",
+  //   storage: "./database.sqlite",
+  //   logging: console.log,
+  // },
   development: {
-    dialect: "sqlite",
-    storage: "./database.sqlite",
-    logging: console.log,
+    dialect: "postgres",
+    host: "localhost",
+    port: 5432,
+    database: "neurojson_dev",
+    username: process.env.DB_USER_LOCAL,
+    password: process.env.DB_PASSWORD_LOCAL,
+    logging: false,
   },
   test: {
     dialect: "sqlite",

backend/migrations/20260507145253-create-search-tables.js

@@ -0,0 +1,130 @@
+"use strict";
+
+/** @type {import('sequelize-cli').Migration} */
+module.exports = {
+  async up(queryInterface, Sequelize) {
+    // ioviews table
+    await queryInterface.createTable("ioviews", {
+      id: {
+        type: Sequelize.INTEGER,
+        autoIncrement: true,
+        primaryKey: true,
+        allowNull: false,
+      },
+      dbname: {
+        type: Sequelize.STRING(30),
+        allowNull: true,
+      },
+      dsname: {
+        type: Sequelize.STRING(30),
+        allowNull: true,
+      },
+      subj: {
+        type: Sequelize.STRING(12),
+        allowNull: true,
+      },
+      view: {
+        type: Sequelize.STRING(12),
+        allowNull: true,
+      },
+      json: {
+        type: Sequelize.JSONB,
+        allowNull: true,
+      },
+      search_vector: {
+        type: Sequelize.DataTypes.TSVECTOR,
+        allowNull: true,
+      },
+      updated_at: {
+        type: Sequelize.DATE,
+        allowNull: false,
+        defaultValue: Sequelize.literal("CURRENT_TIMESTAMP"),
+      },
+    });
+
+    // ioviews indexes
+    await queryInterface.addIndex("ioviews", ["view"], {
+      name: "idx_ioviews_view",
+    });
+    await queryInterface.addIndex("ioviews", ["dbname"], {
+      name: "idx_ioviews_dbname",
+    });
+    await queryInterface.addIndex("ioviews", ["updated_at"], {
+      name: "idx_ioviews_updated_at",
+    });
+
+    // GIN indexes need raw query (not supported by addIndex)
+    await queryInterface.sequelize.query(`
+    CREATE INDEX IF NOT EXISTS idx_ioviews_search 
+      ON ioviews USING GIN(search_vector);
+    CREATE INDEX IF NOT EXISTS idx_ioviews_json 
+      ON ioviews USING GIN(json);
+  `);
+
+    // iolinks table
+    await queryInterface.createTable("iolinks", {
+      id: {
+        type: Sequelize.INTEGER,
+        autoIncrement: true,
+        primaryKey: true,
+        allowNull: false,
+      },
+      dbname: {
+        type: Sequelize.STRING(30),
+        allowNull: true,
+      },
+      dsname: {
+        type: Sequelize.STRING(30),
+        allowNull: true,
+      },
+      subj: {
+        type: Sequelize.TEXT,
+        allowNull: true,
+      },
+      view: {
+        type: Sequelize.TEXT,
+        allowNull: true,
+      },
+      json: {
+        type: Sequelize.JSONB,
+        allowNull: true,
+      },
+    });
+
+    // iolinks indexes
+    await queryInterface.addIndex("iolinks", ["view"], {
+      name: "idx_iolinks_view",
+    });
+    await queryInterface.addIndex("iolinks", ["dbname"], {
+      name: "idx_iolinks_dbname",
+    });
+    await queryInterface.sequelize.query(`
+    CREATE INDEX IF NOT EXISTS idx_iolinks_json 
+      ON iolinks USING GIN(json);
+  `);
+
+    // sync_state table
+    await queryInterface.createTable("sync_state", {
+      dbname: {
+        type: Sequelize.STRING(30),
+        primaryKey: true,
+        allowNull: false,
+      },
+      last_seq: {
+        type: Sequelize.TEXT,
+        allowNull: true,
+      },
+      synced_at: {
+        type: Sequelize.DATE,
+        allowNull: false,
+        defaultValue: Sequelize.literal("CURRENT_TIMESTAMP"),
+      },
+    });
+  },
+
+  async down(queryInterface, Sequelize) {
+    await queryInterface.dropTable("ioviews");
+    await queryInterface.dropTable("iolinks");
+    await queryInterface.dropTable("sync_state");
+  },
+};

backend/migrations/20260508195500-add-ioviews-unique-constraint.js

@@ -0,0 +1,20 @@
+"use strict";
+
+/** @type {import('sequelize-cli').Migration} */
+module.exports = {
+  async up(queryInterface, Sequelize) {
+    // Required by upsertIoview's ON CONFLICT (dbname, dsname, subj, view).
+    await queryInterface.addConstraint("ioviews", {
+      fields: ["dbname", "dsname", "subj", "view"],
+      type: "unique",
+      name: "ioviews_dbname_dsname_subj_view_unique",
+    });
+  },
+
+  async down(queryInterface, Sequelize) {
+    await queryInterface.removeConstraint(
+      "ioviews",
+      "ioviews_dbname_dsname_subj_view_unique"
+    );
+  },
+};

backend/migrations/20260511145900-widen-ioviews-iolinks-text-columns.js

@@ -0,0 +1,60 @@
+"use strict";
+
+/** @type {import('sequelize-cli').Migration} */
+module.exports = {
+  async up(queryInterface, Sequelize) {
+    // VARCHAR(n) → TEXT is a metadata-only change in Postgres (no table rewrite,
+    // no need to drop the unique constraint or indexes).
+    await queryInterface.changeColumn("ioviews", "dbname", {
+      type: Sequelize.TEXT,
+      allowNull: true,
+    });
+    await queryInterface.changeColumn("ioviews", "dsname", {
+      type: Sequelize.TEXT,
+      allowNull: true,
+    });
+    await queryInterface.changeColumn("ioviews", "subj", {
+      type: Sequelize.TEXT,
+      allowNull: true,
+    });
+    await queryInterface.changeColumn("iolinks", "dbname", {
+      type: Sequelize.TEXT,
+      allowNull: true,
+    });
+    await queryInterface.changeColumn("iolinks", "dsname", {
+      type: Sequelize.TEXT,
+      allowNull: true,
+    });
+    await queryInterface.changeColumn("sync_state", "dbname", {
+      type: Sequelize.TEXT,
+      allowNull: false,
+    });
+  },
+
+  async down(queryInterface, Sequelize) {
+    await queryInterface.changeColumn("ioviews", "dbname", {
+      type: Sequelize.STRING(30),
+      allowNull: true,
+    });
+    await queryInterface.changeColumn("ioviews", "dsname", {
+      type: Sequelize.STRING(30),
+      allowNull: true,
+    });
+    await queryInterface.changeColumn("ioviews", "subj", {
+      type: Sequelize.STRING(12),
+      allowNull: true,
+    });
+    await queryInterface.changeColumn("iolinks", "dbname", {
+      type: Sequelize.STRING(30),
+      allowNull: true,
+    });
+    await queryInterface.changeColumn("iolinks", "dsname", {
+      type: Sequelize.STRING(30),
+      allowNull: true,
+    });
+    await queryInterface.changeColumn("sync_state", "dbname", {
+      type: Sequelize.STRING(30),
+      allowNull: false,
+    });
+  },
+};

backend/package.json

@@ -11,7 +11,8 @@
     "migrate:undo:all": "npx sequelize-cli db:migrate:undo:all",
     "seed": "npx sequelize-cli db:seed:all",
     "seed:undo": "npx sequelize-cli db:seed:undo:all",
-    "db:reset": "npx sequelize-cli db:migrate:undo:all && npx sequelize-cli db:migrate"
+    "db:reset": "npx sequelize-cli db:migrate:undo:all && npx sequelize-cli db:migrate",
+    "sync": "node sync/incrementalSync.js"
   },
   "keywords": [
     "express",