From aee91c4ed0277e343a62cd124ee7175f54ffefb3 Mon Sep 17 00:00:00 2001 From: Linux User Date: Sat, 22 Jul 2023 16:17:55 +0300 Subject: [PATCH] wip - main, config, ctlfs --- .gitignore | 2 + LICENSE | 27 ++++++ README | 32 +++++++ appl/cmd/config.b | 118 ++++++++++++++++++++++++ appl/cmd/ctlfs.b | 222 ++++++++++++++++++++++++++++++++++++++++++++++ appl/cmd/dddb.b | 94 ++++++++++++++++++++ appl/cmd/mkfile | 18 ++++ appl/lib/mkfile | 11 +++ appl/mkfile | 6 ++ mkconfig | 6 ++ mkfile | 6 ++ 11 files changed, 542 insertions(+) create mode 100644 .gitignore create mode 100644 LICENSE create mode 100644 README create mode 100644 appl/cmd/config.b create mode 100644 appl/cmd/ctlfs.b create mode 100644 appl/cmd/dddb.b create mode 100644 appl/cmd/mkfile create mode 100644 appl/lib/mkfile create mode 100644 appl/mkfile create mode 100644 mkconfig create mode 100644 mkfile diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..6265895 --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +*.dis +*.sbl diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..7ab574f --- /dev/null +++ b/LICENSE @@ -0,0 +1,27 @@ +Copyright © 2023, kitzman @ disroot.org + +All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + * Neither the name of {{ project }} nor the names of its contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. \ No newline at end of file diff --git a/README b/README new file mode 100644 index 0000000..564575e --- /dev/null +++ b/README @@ -0,0 +1,32 @@ +dddb (Distributed DisVM Database) is a WIP RDBMS, aimed +to run on DisVM, hosted or native. + +TODO Components and functionalities + +* dddbctl fs - authenticated 9p fs to control, request, and query + operations related to the database instance; it + is the main driver for grid database +* dddbclient fs - the tcp port 'dddb' is used such that clients + can have a fs in which queries can be opened; + it will support ANSI SQL +* db registry - each instance will, on startup, connect to + other instances defined in an ndb configuration + file, register itself in that instance, and + will be able to use it for: duplicating data, + sharding data, and mount further instances; + (i.e if a cluster is behind a firewall, and + only one instance exposed, the hidden ones + can be mounted outside the fw) +* data structures (1) - primitives, pages, sections, indices, + tables, rows +* data structures (2) - B-trees, Bloom filters +* storage operations - a mounted directory tree will be used + for RDRW operations; memfs will be used + for caching the data in RAM (performance?) +* venti/vac support - venti + vac + memfs should suffice for + storing and persisting data (ramfossil-like); + this means scripts will clean and archive it +* SQL parser - obviously +* statement optimizer - obviously +* wm/dddbmon - a nice admin monitor would be nice +* dddbjdbc/dddbodbc - JDBC and ODBC drivers are mandatory diff --git a/appl/cmd/config.b b/appl/cmd/config.b new file mode 100644 index 0000000..57f28cb --- /dev/null +++ b/appl/cmd/config.b @@ -0,0 +1,118 @@ +include "bufio.m"; +include "attrdb.m"; + attrdb: Attrdb; + Attr, Db, Dbptr, Dbentry: import attrdb; + +# Default configuration values +DCFGPATH: con "/lib/ndb/dddbcfg"; + +DADDR: con "tcp!*!dddbctl"; +DFSWRKS: con 10; + +# Attrdb constants +KNAME: con "nodename"; +KSYSNAME: con "nodesysn"; +KADDR: con "addr"; +KSTORAGE: con "storage"; +KFSWRKS: con "readworkers"; + +Config.open(nodename: string, path: string): Config +{ + sys = load Sys Sys->PATH; + + n: int; + sysname: string; + buf := array[Sys->NAMEMAX] of byte; + + attrdb = load Attrdb Attrdb->PATH; + if(attrdb == nil) + error("Attrb not found"); + attrdb->init(); + + if(len path == 0) + path = DCFGPATH; + + if(debug) + sys->fprint(stderr, "config: opening %s\n", path); + db := Db.open(path); + + if(db == nil) + error(sys->sprint("ndb file %s could not be opened", path)); + + sysname_fd := sys->open("#c/sysname", Sys->OREAD); + if(sysname_fd == nil) + error(sys->sprint("#c/sysname could not be read")); + n = sys->read(sysname_fd, buf, len buf); + if(n <= 0) + error(sys->sprint("could not read sysname")); + sysname = string buf[0:n]; + + if(len nodename == 0) { + nodename = sysname; + } + + thiscfg: ref NodeConfig; + nodecfgs: list of NodeConfig; + entry: ref Dbentry; + dbptr: ref Dbptr; + + (entry, dbptr) = db.find(dbptr, KNAME); + while(entry != nil) { + nodecfg := NodeConfig.new(entry); + + if(debug) + sys->fprint(stderr, "config: found node %s\n", nodecfg.name); + if(nodecfg.name == nodename) { + thiscfg = ref nodecfg; + } + else + nodecfgs = nodecfg :: nodecfgs; + + (entry, dbptr) = db.find(dbptr, KNAME); + } + + if(thiscfg == nil) + error("could not find config for nodename"); + + if(thiscfg.addr == "") + error("node lacks an address"); + + if(thiscfg.storage == "") + error("node lacks a storage path"); + + if(thiscfg.sysn != sysname) + error("node and system sysname do not match"); + + return Config( + thiscfg.name, thiscfg.sysn, thiscfg.addr, # own configuration + thiscfg.storage, thiscfg.fswrks, + nodecfgs); # configured nodes +} + +NodeConfig.new(entry: ref Dbentry): NodeConfig +{ + name := entry.findfirst(KNAME); + sysname := entry.findfirst(KSYSNAME); + addr := entry.findfirst(KADDR); + storage := entry.findfirst(KSTORAGE); + fswrks_s := entry.findfirst(KFSWRKS); + + fswrks := DFSWRKS; + + if(len name == 0) + error("entry has no name"); + if(len sysname == 0) + error("entry has no sysname"); + if(len addr == 0) + addr = DADDR; + if(len fswrks_s != 0) { + (fswrks_i, rm) := strm->toint(fswrks_s, 10); + if(rm != "") + error(sys->sprint("malformed fs workers count: %s", fswrks_s)); + fswrks = fswrks_i; + } + + return NodeConfig( + name, sysname, addr, storage, # basic information + fswrks); # tunable options +} diff --git a/appl/cmd/ctlfs.b b/appl/cmd/ctlfs.b new file mode 100644 index 0000000..30a94bb --- /dev/null +++ b/appl/cmd/ctlfs.b @@ -0,0 +1,222 @@ +include "dial.m"; + dial: Dial; + +include "security.m"; + auth: Auth; + +include "styx.m"; + styx: Styx; + Tmsg, Rmsg: import Styx; + +include "styxservers.m"; + styxservers: Styxservers; + Styxserver, Fid, Navigator, + Navop, Enotfound, Enotdir: import styxservers; + +# FS file index +Qroot, Qctl, Qstats, Qmax: con iota; +tab := array[] of { + (Qroot, ".", Sys->DMDIR|8r555), + (Qctl, "ctl", 8r222), + (Qstats, "stats", 8r111), +}; + +# create ctlfs and the appropriate listeners +init_ctlfs(cfg: Config, keyfile: string, algs: list of string) +{ + dial = load Dial Dial->PATH; + auth = load Auth Auth->PATH; + styx = load Styx Styx->PATH; + + if(dial == nil) + error("ctlfs: dial module not found"); + if(auth == nil) + error("ctlfs: auth module not found"); + if(styx == nil) + error("ctlfs: styx module not found"); + + auth->init(); + + styx->init(); + styxservers->init(styx); + styxservers->traceset(chatty); + + # authinfo init + if(debug) + sys->fprint(stderr, "ctlfs: reading authinfo"); + authinfo: ref Keyring->Authinfo; + if (doauth) { + if (keyfile == nil) + keyfile = "/usr/" + user() + "/keyring/default"; + authinfo = keyring->readauthinfo(keyfile); + if (authinfo == nil) + error(sys->sprint("ctlfs: cannot read %s: %r", keyfile)); + } + + # announcing + if(debug) + sys->fprint(stderr, "ctlfs: announcing dddbctl"); + addr := dial->netmkaddr(cfg.addr, "tcp", "dddbctl"); + c := dial->announce(addr); + if(c == nil) + error(sys->sprint("ctlfs: cannot listen on %s\n", addr)); + + # bootstrapping + if(debug) + sys->fprint + sys->unmount(nil, "/mnt/keys"); + + navch := chan of ref Navop; + spawn ctlfs_navigator(navch); + + nav := Navigator.new(navch); + (tc, srv) := Styxserver.new(fildes(0), nav, big Qroot); + + # listener entrypoint + listener(c, authinfo, algs); +} + +# dddbctl listener loop +ctlfs_listener(c: ref Dial->Connection, authinfo: ref Keyring->Authinfo, algs: list of string) +{ + for (;;) { + nc := dial->listen(c); + if (nc == nil) + error(sys->sprint("listen failed: %r")); + if (debug) + sys->fprint(stderr, "ctlfs: got connection from %s\n", + readfile(nc.dir + "/remote")); + dfd := dial->accept(nc); + if (dfd != nil) { + if(nc.cfd != nil) + sys->fprint(nc.cfd, "keepalive"); + hostname: string; + if(passhostnames){ + hostname = readfile(nc.dir + "/remote"); + if(hostname != nil) + hostname = hostname[0:len hostname - 1]; + } + + spawn ctlfs_authenticator(dfd, authinfo, algs, hostname); + } + } +} + +# authenticate a connection and set the user id. +ctlfs_authenticator(dfd: ref Sys->FD, authinfo: ref Keyring->Authinfo, + algs: list of string, hostname: string) +{ + # authenticate and change user id appropriately + (fd, err) := auth->server(algs, authinfo, dfd, 1); + if (fd == nil) { + if (debug) + sys->fprint(stderr(), "ctlfs: authentication failed: %s\n", err); + return; + } + if (debug) + sys->fprint(stderr(), "ctlfs: client authenticated as %s\n", err); + + spawn exportproc(sync, mfd, err, hostname, fd); +} + +ctlfs_loop() +{ + # Primary server loop + loop: + while((tmsg := <-tc) != nil) { + # Switch on operations being performed on a given Fid + pick msg := tmsg { + Open => + srv.default(msg); + Read => + fid := srv.getfid(msg.fid); + + if(fid.qtype & Sys->QTDIR) { + # This is a directory read + srv.default(msg); + continue loop; + } + + case int fid.path { + Qlog => + # A read on our log file, tell them what they've already said ? + s := ""; + + for(l := log; l != nil; l = tl l) + s = hd l + s; + + srv.reply(styxservers->readstr(msg, s)); + + * => + srv.default(msg); + } + + Write => + fid := srv.getfid(msg.fid); + + case int fid.path { + Qctl => + # Don't care about offset + cmd := string msg.data; + + reply: ref Rmsg = ref Rmsg.Write(msg.tag, len msg.data); + + case cmd { + * => + # Ignore empty writes + if(cmd != nil) + log = cmd :: log; + else + reply = ref Rmsg.Error(msg.tag, "empty write!"); + } + srv.reply(reply); + + * => + srv.default(msg); + } + + * => + srv.default(msg); + } + } + + exit; +} + +# Navigator function for moving around under / +ctlfs_navigator(c: chan of ref Navop) { + loop: + for(;;) { + navop := <-c; + pick op := navop { + Stat => + op.reply <-= (dir(int op.path), nil); + + Walk => + if(op.name == "..") { + op.reply <-= (dir(Qroot), nil); + continue loop; + } + + case int op.path&16rff { + + Qroot => + for(i := 1; i < Qmax; i++) + if(tab[i].t1 == op.name) { + op.reply <-= (dir(i), nil); + continue loop; + } + + op.reply <-= (nil, Enotfound); + * => + op.reply <-= (nil, Enotdir); + } + + Readdir => + for(i := 0; i < op.count && i + op.offset < (len tab) - 1; i++) + op.reply <-= (dir(Qroot+1+i+op.offset), nil); + + op.reply <-= (nil, nil); + } + } +} diff --git a/appl/cmd/dddb.b b/appl/cmd/dddb.b new file mode 100644 index 0000000..d3f7c46 --- /dev/null +++ b/appl/cmd/dddb.b @@ -0,0 +1,94 @@ +implement Dddb; + +include "sys.m"; + sys: Sys; +include "arg.m"; +include "draw.m"; +include "string.m"; + strm: String; + +include "config.b"; +include "ctlfs.b"; + +stderr: ref Sys->FD; +debug: int; + +error(s: string) +{ + sys->fprint(stderr, "dddb: %s\n", s); + raise "dddb:error"; +} + +Dddb: module { + init: fn(nil: ref Draw->Context, args: list of string); + run_fs: fn(cfg: Config); + + Config: adt { + name: string; + sysn: string; + addr: string; + storage: string; + fswrks: int; + nodes: list of NodeConfig; + + open: fn(nodename: string, path: string): Config; + }; + + NodeConfig: adt { + name: string; + sysn: string; + addr: string; + storage: string; + fswrks: int; + + new: fn(entry: ref Dbentry): NodeConfig; + }; + +}; + +init(nil: ref Draw->Context, args: list of string) +{ + sys = load Sys Sys->PATH; + arg := load Arg Arg->PATH; + strm = load String String->PATH; + + stderr = sys->fildes(2); + cfgpath: string = ""; + + arg->init(args); + arg->setusage(arg->progname()+ " [-d] [-c config] nodename"); + while((c := arg->opt()) != 0) + case c { + 'd' => debug++; + 'c' => + cfgpath = arg->earg(); + * => + sys->fprint(sys->fildes(2), "bad option: -%c\n", c); + arg->usage(); + } + + args = arg->argv(); + + nodename := hd args; + + if(nodename == nil) { + sys->fprint(stderr, "dddb: no nodename supplied\n"); + arg->usage(); + } + + if(debug) + sys->fprint(stderr, "dddb: opening config file\n"); + cfg := Config.open(nodename, cfgpath); + + if(debug) { + sys->fprint(stderr, "dddb: database parms:\n"); + sys->fprint(stderr, "cfg.name: %s\n", cfg.name); + sys->fprint(stderr, "cfg.sysn: %s\n", cfg.sysn); + sys->fprint(stderr, "cfg.storage: %s\n", cfg.storage); + sys->fprint(stderr, "cfg.fswrks: %d\n", cfg.fswrks); + } + + run_fs(cfg); +} + + diff --git a/appl/cmd/mkfile b/appl/cmd/mkfile new file mode 100644 index 0000000..ffb1bbd --- /dev/null +++ b/appl/cmd/mkfile @@ -0,0 +1,18 @@ +<../../mkconfig + +TARG=\ + dddb.dis\ + +MODULES=\ + +SYSMODULES=\ + arg.m\ + dial.m\ + draw.m\ + sys.m\ + styx.m\ + styxservers.m\ + +DISBIN=$home/dis + +<$ROOT/mkfiles/mkdis diff --git a/appl/lib/mkfile b/appl/lib/mkfile new file mode 100644 index 0000000..3fd6ec4 --- /dev/null +++ b/appl/lib/mkfile @@ -0,0 +1,11 @@ +<../../mkconfig + +TARG=\ + +MODULES=\ + +SYSMODULES=\ + +DISBIN=$ROOT/dis/lib + +<$ROOT/mkfiles/mkdis diff --git a/appl/mkfile b/appl/mkfile new file mode 100644 index 0000000..392dbf6 --- /dev/null +++ b/appl/mkfile @@ -0,0 +1,6 @@ +<../mkconfig + +DIRS=\ + cmd\ + +<$ROOT/mkfiles/mksubdirs diff --git a/mkconfig b/mkconfig new file mode 100644 index 0000000..9cd9071 --- /dev/null +++ b/mkconfig @@ -0,0 +1,6 @@ +<$ROOT/mkconfig + +# Comment the below lines for hosted builds +SYSHOST=Inferno +SYSTARG=Inferno +ROOT=/ diff --git a/mkfile b/mkfile new file mode 100644 index 0000000..a1e7d95 --- /dev/null +++ b/mkfile @@ -0,0 +1,6 @@ +