Skip to content
Draft

Db next #1117

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
56 commits
Select commit Hold shift + click to select a range
d0627ba
separate test and testing files
ionous Oct 11, 2026
5106a1c
db v2
ionous Oct 14, 2026
d7c0a6e
Update TABLES.md
ionous Oct 16, 2026
f5e4dce
db-next: migration tool for new tables
ionous Oct 16, 2026
1febe51
Revert "db-next: migration tool for new tables"
ionous Oct 16, 2026
6541677
update to match the standalone migration tool 1/3
ionous Oct 20, 2026
02b5227
update to match the standalone migration tool 2/3
ionous Oct 20, 2026
fded059
update to match the standalone migrationtool 3/3
ionous Oct 20, 2026
ec05ef6
two view fixes:
ionous Oct 21, 2026
01dc16d
restore use of app path and add mysql port override
ionous Oct 21, 2026
8c9c32e
... and restore the package kson paths too
ionous Oct 21, 2026
3c122db
migration fixes
ionous Oct 21, 2026
7e8b756
use joined view for queries
ionous Oct 21, 2026
9ffddbe
fix retrieve event to return days of not yet published events
ionous Oct 23, 2026
15d12a4
Delete test.db
ionous Oct 23, 2026
d9beda8
update new tables doc as "TABLES-V2.md"
ionous Oct 23, 2026
f41580e
Merge remote-tracking branch 'upstream/main'
ionous Oct 23, 2026
05c2442
Merge remote-tracking branch 'upstream/main' into db-next
ionous Oct 25, 2026
1addfc7
merged rom main
ionous Oct 25, 2026
e616487
fix ridelength default value; add tests for that.
ionous Oct 25, 2026
6827874
fix web print default value
ionous Oct 25, 2026
b089386
git ignore .db files
ionous Oct 25, 2026
9bd2a65
migration: ensure created time is less than modified
ionous Oct 26, 2026
06c8627
migration: fix missing locend becoming the string "null"
ionous Oct 26, 2026
7653902
migration: make notes about null emails
ionous Oct 26, 2026
0938ab5
fix: don't allow rides with null secrets to be updated
ionous Oct 26, 2026
214eeb8
remove lowdash dependency
ionous Oct 26, 2026
10beb13
collapse getFullRange and getRangeVisible
ionous Oct 26, 2026
542c3ab
use node assertions to validate parameters
ionous Oct 27, 2026
cb7862e
tweak to test name for clarity
ionous Oct 27, 2026
ab7b464
fix: handle thrown exceptions better
ionous Oct 27, 2026
a5cbf81
node assert instead of console assert
ionous Oct 27, 2026
054b110
tweaks to make docs match endpoints and vice versa
ionous Oct 27, 2026
46fdfe0
ensure event endtime reports 24 time
ionous Oct 27, 2026
62d5bda
improve test names
ionous Oct 27, 2026
666ddc9
improve image docs, and remove app/eventimages
ionous Oct 27, 2026
2edbaf4
handle tinytitle differences
ionous Oct 28, 2026
7cf4efa
fix fake events generator
ionous Nov 1, 2026
4577e65
make shorthand code a little easier to understand
ionous Nov 1, 2026
35aa22a
handle deleted events and delisted days
ionous Nov 1, 2026
11ab042
migration: set revoked to null instead of undefined
ionous Nov 6, 2026
26b7200
set mysql collation of eventstatus
ionous Nov 6, 2026
ecfa6b6
Merge remote-tracking branch 'upstream/main'
ionous Nov 6, 2026
224f01e
migration from the db-next branch
ionous Nov 6, 2026
819b980
Update TABLES-V2.md
ionous Nov 6, 2026
bc0ebaf
add an initial connection collation; comments
ionous Nov 6, 2026
cbcbf01
switch to using "hasOwn" from "in"
ionous Nov 12, 2026
e55fc4c
merge from migration changes branch
ionous Nov 15, 2026
cb01c02
fixup package dependencies
ionous Nov 15, 2026
60ca9aa
fixes for email, phone, contact fields
ionous Nov 19, 2026
aecb847
trim auto-generated tinytitle
ionous Nov 19, 2026
5c97c0e
fix migration of loopride data
ionous Nov 19, 2026
c08189e
fix show web, and manage event unit test failure
ionous Nov 19, 2026
f1ede96
Merge remote-tracking branch 'upstream/main'
ionous Nov 19, 2026
4bbf6e3
Merge branch 'main' into db-next
ionous Nov 19, 2026
369ee43
finish merge
ionous Nov 19, 2026
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 6 additions & 7 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -6,23 +6,22 @@ shift.overrides
.idea/
!services/db/*
*.sql
*.db
!services/db/*.sql
!services/db/*/*.sql
services/db/tmp/*.sql
*.today
*.DS_Store
# this is the current location of event image file upload
# (ex. on production and during local docker development)
backend/eventimages/
secrets
services/php/shift-mail.log
services/node/shift-mail.log
app/node_modules/
# exclude all event images except the readme and test bike image.
app/eventimages/*
!app/eventimages/
!app/eventimages/README.md
!app/eventimages/bike.jpg
!app/eventimages/bike-big.png
!app/eventimages/bike-bad.tiff
# once upon a time, this was where event images were uploaded.
# therefore, exclude everything from this directory.
app/eventimages/
bin/
node_modules/
/frontend/.vscode
Expand Down
8 changes: 4 additions & 4 deletions app/app.js
Original file line number Diff line number Diff line change
Expand Up @@ -5,20 +5,20 @@ const config = require('./config');
const { initMail } = require( './emailer');
const app = require( './appEndpoints');
const db = require('./db'); // initialize on startup
const tables = require("./models/tables");
const schema = require('shift-docs/schema');

// connect to the db
db.initialize().then(async () => {
// create db tables
await tables.createTables();
await schema.setupTables(db);

// connect to the smtp server
await initMail().then(hostName => {
console.log("okay: verified email host", hostName);
}).catch(e => {
console.error("failed smtp verification because", e.toString());
console.warn("Warning: Sending email will not work because", e.toString());
}).finally(_ => {
console.log("and emails will log to", config.email.logfile || "console");
console.log("Emails will log to", config.email.logfile || "console");
});

// start a webserver to listen to all requests
Expand Down
147 changes: 107 additions & 40 deletions app/config.js
Original file line number Diff line number Diff line change
@@ -1,13 +1,40 @@
const path = require('path');
const fs = require("fs");
/**
* Global configuration for the backend.
* Holds all environment, commandline options, paths, etc.
*/
const fs = require('node:fs'); // for reading email config
const os = require('node:os'); // for tmpdir fallbacks
const path = require('node:path');
const CommandLine = require('shift-docs/util/cmdLine');

// helper to read environment variables
function env_default(field, def) {
return process.env[field] ?? def;
}

// node server listen for its http requests
// fix? there is no such environment variable right now.
const listen = env_default('NODE_PORT', 3080);
const cmdLine = new CommandLine({
db_debug: [
"Can be either 'true' or 'false'.",
" Defaults to 'false'. When 'true', adds extra debugging info for queries.",
],
db: [
"Can be either 'sqlite' or 'mysql'.",
"For prod and dev it defaults to 'mysql'.",
"For tests, it defaults to 'sqlite'.",
"- 'mysql': Requires docker.",
" For prod and dev it pulls its full configuration from the environment.",
" Tests have a hardcoded configuration.",
" This option can followed by a localhost port, '-db=mysql:61530'",
"- 'sqlite': Uses a sqlite database.",
" This option can followed by the path to its file.",
" The default is equivalent to specifying: '-db=sqlite:./bin/events.db'.",
],
});

// determine the port for the node server.
const listen = (() => {
return 3080;
})();

// the user facing server.
const siteHost = siteUrl(listen);
Expand All @@ -18,20 +45,38 @@ const appPath = path.resolve(__dirname);
// for max file size
const bytesPerMeg = 1024*1024;

const staticFiles = env_default('SHIFT_STATIC_FILES');
// ex. npm_lifecycle_event: 'dev-setup'
const isTesting = process.env.npm_lifecycle_event && process.env.npm_lifecycle_event.startsWith('test');

// optional directory containing front-end content.
// if it doesnt exist, assumes the app isn't trying to serve static files.
// ( and that there is a separate front-end running out there, ex. production. )
const staticFiles = (() => {
const p = env_default('SHIFT_STATIC_FILES');
return !p ? null : path.resolve(appPath, p);
})();

const isTesting = !!(process.env.npm_lifecycle_event || "").match(/test$/);
// read the command line parameter for db configuration
const dbType = env_default('npm_config_db');
const dbDebug = !!env_default('npm_config_db_debug');
// storage location for uploaded images.
// forces a temp directory when running unit tests.
const imageDestination = (() => {
const p = env_default('SHIFT_IMAGE_DIR');
if (isTesting) {
return os.tmpdir();
} else if (p) {
return path.resolve(appPath, p);
}
})();

const dbConfig = getDatabaseConfig(cmdLine.options.db, cmdLine.bool('db_debug'), isTesting);

// The config object exported
const config = {
appPath,
api: {
header: 'Api-Version',
version: "3.65.0",
version: "3.66.0",
},
db: getDatabaseConfig(dbType, isTesting),
db: dbConfig,
// maybe bad, but some code likes to know:
isTesting,
// a nodemailer friendly config, or false if smtp is not configured.
Expand All @@ -40,7 +85,7 @@ const config = {
name: "SHIFT to Bikes",
listen,
path: "/",
// used for shareable links, and the manage url sent in email
// used for crawl url, shareable links, and the manage url sent in email
url(...parts) {
const base = `${siteHost}${config.site.path}`;
return base + parts.join("/");
Expand All @@ -49,6 +94,10 @@ const config = {
return config.site.url("pages/calendar-faq/");
},
staticFiles,
// expose frontend content for adding and viewing events.
// in production: these endpoints are served via netfliy.
// uses path.posix to get forward slashes even on windows
// i forget exactly why...
devEndpoints: !staticFiles ? null : [{
// ex. http://localhost:3080/addevent/edit-1-d00c888b0a1d4bab8107ba2fbe2beddf
// loads http://localhost:3080/addevent/index.html
Expand Down Expand Up @@ -88,8 +137,7 @@ const config = {
},
image: {
// storage location for event images
dir: path.resolve(appPath,
env_default('SHIFT_IMAGE_DIR', "eventimages")),
dir: imageDestination,
// used for generated event image links
// ( see also the ngnix config )
path: env_default('SHIFT_IMAGE_PATH', "/eventimages/"),
Expand Down Expand Up @@ -142,7 +190,7 @@ module.exports = config;
// determine the user facing prefix for generating page and image links.
// production - https://shift2bikes.org
// docker dev - https://localhost:4443
// standalone or test - http://localhost:3080
// npm dev, npm test - http://localhost:3080
function siteUrl(proxyPort) {
const hostname = env_default('SHIFT_DOMAIN', 'localhost');
// production uses 443, docker dev uses 4443, standalone is null.
Expand Down Expand Up @@ -190,23 +238,18 @@ function getSmtpSettings() {
}

// our semi-agnostic database configuration
function getDatabaseConfig(dbType, isTesting) {
// dbType comes from the command-line
// if nothing was specfied, use the MYSQL_DATABASE environment variable
const env = env_default('MYSQL_DATABASE')
if (!dbType && env) {
dbType = env.startsWith("sqlite") ? env : null;
}
if (!dbType) {
dbType = isTesting ? 'sqlite' : 'mysql'
}
function getDatabaseConfig(dbType, dbDebug, isTesting) {
dbType = dbType || readDbEnv(isTesting ? 'sqlite' : null);

const [name, parts] = dbType.split(':');
const config = {
mysql: !isTesting ? getMysqlDefault : getMysqlTesting,
sqlite: getSqliteConfig,
}
if (!name in config) {
throw new Error(`unknown database type '${dbType}'`)
if (!Object.hasOwn(config, name)) {
const types = Object.keys(config).join(", ");
const unknownConfig = `unknown database type '${dbType}'. expected one of: ${types}.`;
throw new Error(unknownConfig);
}
return {
type: name,
Expand All @@ -215,22 +258,46 @@ function getDatabaseConfig(dbType, isTesting) {
}
}

// the default for mysql when running dev or production
function getMysqlDefault() {
return {
host: env_default('MYSQL_HOST', 'db'),
port: env_default('MYSQL_PORT', 3306), // standard mysql port.
user: env_default('MYSQL_USER', 'shift'),
pass: env_default('MYSQL_PASSWORD', 'ok124'),
name: env_default('MYSQL_DATABASE', 'shift'),
// nothing specified in the env fails
// sqlite specified in the env uses.... sqlite.
// anything other than sqlite in the env is assumed to be a mysql spec.
function readDbEnv(fallback, env = 'MYSQL_DATABASE') {
const dbType = env_default(env) || fallback;
if (!dbType) {
throw new Error(`expected db type on command line or '${env}' env variable`);
}
return dbType.startsWith("sqlite") ? dbType : 'mysql';
}

// the default for mysql when running dev or production
// 3306 standard mysql port.
function getMysqlDefault(port) {
const common = {
user: 'shift',
pass: 'ok124',
name: 'shift',
};
const env = {
user: env_default('MYSQL_USER'),
pass: env_default('MYSQL_PASSWORD'),
name: env_default('MYSQL_DATABASE'),
host: env_default('MYSQL_HOST', "db"),
port: env_default('MYSQL_PORT', 3306),
};
// normally, a port isnt assigned via the command line
// in that case, the environment wins over the common settings
// with a port, we ignore the environment
return Object.assign({}, common, !port ? env : {
host: "localhost",
port: port,
});
}

// the default for mysql when running tests
function getMysqlTesting() {
function getMysqlTesting(port) {
return {
host: "localhost",
port: 3308, // custom EXTERNAL port to avoid conflicts
port: port ?? 3308, // custom EXTERNAL port to avoid conflicts
user: 'shift_test',
pass: 'shift_test',
name: 'shift_test',
Expand All @@ -239,9 +306,9 @@ function getMysqlTesting() {

// the default for sqlite
// if filename is null, it uses a memory database
// paths are relative to npm's starting path.
// relative paths are resolved using the shift root directory
function getSqliteConfig(filename) {
const connection = !filename ? ":memory:" : path.resolve(appPath, filename);
const connection = !filename ? ":memory:" : path.resolve(appPath , filename);
return {
name: connection
};
Expand Down
75 changes: 7 additions & 68 deletions app/db.js
Original file line number Diff line number Diff line change
@@ -1,12 +1,10 @@
const knex = require('knex'); // the knex constructor
const pickBy = require('lodash/pickBy'); // a dependency of package knex
const config = require("./config");
const dt = require("./util/dateTime");
const config = require('shift-docs/config');
const dt = require('shift-docs/util/dateTime');

// build knex configuration from our own
const dbConfig = unpackConfig(config.db);
const useSqlite = config.db.type === 'sqlite';
const dropOnCreate = config.db.connect?.name === 'shift_test';

const db = {
config: config.db,
Expand All @@ -17,12 +15,13 @@ const db = {
query: false,

// waits to open a connection.
async initialize(name) {
async initialize(debugName = "unknown") {
if (db.query) {
throw new Error("db already initialized");
throw new Error(`db being initialized by ${debugName} when already initialized by ${this.debugName}.`);
}
const connection = knex(dbConfig);
db.query = connection;
db.debugName = debugName;
await connection;
},

Expand All @@ -33,77 +32,17 @@ const db = {
throw new Error("db already destroyed");
}
db.query = false;
db.debugName = `destroyed ${db.debugName}`;
return connection.destroy();
},

// helper shortcut to make raw queries easier to write
raw(...args) {
return db.query.raw(...args);
},

// convert a dayjs object to a 'date' column.
//
// the sqlite driver stores a date as a number
// re: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Date
// while that works, it's hard to read.
//
// mysql stores a date as "YYYY-MM-DD"
// re: https://dev.mysql.com/doc/refman/8.0/en/date-and-time-literals.html
// but the code has been handing it javascript dates, and it seems to like that.
// tbd: test if mysql can (also) use strings; if so, remove the if statement here.
//
toDate(date) {
return !useSqlite ? date.toDate() : dt.toYMDString(date);
},

// sqlite and mysql differ on the keyword.
currentDateString() {
return !useSqlite ? `CURDATE()` : `DATE()`;
},

/**
* update or insert into the database.
* @param table string table name.
* @oaram rec the object containing the data.
* @return promises the rec ( with its new id ).
*
* fix? an orm would probably be smart enough to only update the needed fields.
* this updates *everything*.
*/
store(table, idField, rec) {
const q = db.query(table);
// get everything from that isn't a function()
let cleanData = pickBy(rec, isSafe);
if (rec.exists()) {
// fix: manually set modified for sqlite?
// cleanData.modified = dt.toTimestamp();
return q.update(cleanData)
.where(idField, rec[idField])
.then(_ => rec);
} else {
return q.insert(cleanData)
.then(row => {
rec[idField] = row[0];
return rec;
});
}
},
/**
* delete one (or more) rows from the named table
* where the named field has the value in the passed record.
*/
del(table, idField, rec) {
return db.query(table).where(idField, rec[idField]).del();
},
};
module.exports = db;

// ugh. if knex sees a function in an object,
// it assumes the function generates knex style queries and tries to call them.
// filter them out, mimicking what knex does internally for undefined values.
function isSafe(v, k) {
return (typeof v !== 'function');
}
module.exports = db;

// turn the shift config into knex format
function unpackConfig({ type, connect, debug }) {
Expand Down
Loading