wip - main, config, ctlfs
This commit is contained in:
Linux User
commit
aee91c4ed0
|
|
@ -0,0 +1,2 @@
|
|||
*.dis
|
||||
*.sbl
|
||||
|
|
@ -0,0 +1,27 @@
|
|||
Copyright © 2023, kitzman @ disroot.org
|
||||
|
||||
All rights reserved.
|
||||
|
||||
Redistribution and use in source and binary forms, with or without modification,
|
||||
are permitted provided that the following conditions are met:
|
||||
|
||||
* Redistributions of source code must retain the above copyright notice,
|
||||
this list of conditions and the following disclaimer.
|
||||
* Redistributions in binary form must reproduce the above copyright notice,
|
||||
this list of conditions and the following disclaimer in the documentation
|
||||
and/or other materials provided with the distribution.
|
||||
* Neither the name of {{ project }} nor the names of its contributors
|
||||
may be used to endorse or promote products derived from this software
|
||||
without specific prior written permission.
|
||||
|
||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
||||
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
||||
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
||||
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
|
||||
CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
|
||||
EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
|
||||
PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
|
||||
PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
|
||||
LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
|
||||
NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
|
||||
SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
|
@ -0,0 +1,32 @@
|
|||
dddb (Distributed DisVM Database) is a WIP RDBMS, aimed
|
||||
to run on DisVM, hosted or native.
|
||||
|
||||
TODO Components and functionalities
|
||||
|
||||
* dddbctl fs - authenticated 9p fs to control, request, and query
|
||||
operations related to the database instance; it
|
||||
is the main driver for grid database
|
||||
* dddbclient fs - the tcp port 'dddb' is used such that clients
|
||||
can have a fs in which queries can be opened;
|
||||
it will support ANSI SQL
|
||||
* db registry - each instance will, on startup, connect to
|
||||
other instances defined in an ndb configuration
|
||||
file, register itself in that instance, and
|
||||
will be able to use it for: duplicating data,
|
||||
sharding data, and mount further instances;
|
||||
(i.e if a cluster is behind a firewall, and
|
||||
only one instance exposed, the hidden ones
|
||||
can be mounted outside the fw)
|
||||
* data structures (1) - primitives, pages, sections, indices,
|
||||
tables, rows
|
||||
* data structures (2) - B-trees, Bloom filters
|
||||
* storage operations - a mounted directory tree will be used
|
||||
for RDRW operations; memfs will be used
|
||||
for caching the data in RAM (performance?)
|
||||
* venti/vac support - venti + vac + memfs should suffice for
|
||||
storing and persisting data (ramfossil-like);
|
||||
this means scripts will clean and archive it
|
||||
* SQL parser - obviously
|
||||
* statement optimizer - obviously
|
||||
* wm/dddbmon - a nice admin monitor would be nice
|
||||
* dddbjdbc/dddbodbc - JDBC and ODBC drivers are mandatory
|
||||
|
|
@ -0,0 +1,118 @@
|
|||
include "bufio.m";
|
||||
include "attrdb.m";
|
||||
attrdb: Attrdb;
|
||||
Attr, Db, Dbptr, Dbentry: import attrdb;
|
||||
|
||||
# Default configuration values
|
||||
DCFGPATH: con "/lib/ndb/dddbcfg";
|
||||
|
||||
DADDR: con "tcp!*!dddbctl";
|
||||
DFSWRKS: con 10;
|
||||
|
||||
# Attrdb constants
|
||||
KNAME: con "nodename";
|
||||
KSYSNAME: con "nodesysn";
|
||||
KADDR: con "addr";
|
||||
KSTORAGE: con "storage";
|
||||
KFSWRKS: con "readworkers";
|
||||
|
||||
Config.open(nodename: string, path: string): Config
|
||||
{
|
||||
sys = load Sys Sys->PATH;
|
||||
|
||||
n: int;
|
||||
sysname: string;
|
||||
buf := array[Sys->NAMEMAX] of byte;
|
||||
|
||||
attrdb = load Attrdb Attrdb->PATH;
|
||||
if(attrdb == nil)
|
||||
error("Attrb not found");
|
||||
attrdb->init();
|
||||
|
||||
if(len path == 0)
|
||||
path = DCFGPATH;
|
||||
|
||||
if(debug)
|
||||
sys->fprint(stderr, "config: opening %s\n", path);
|
||||
db := Db.open(path);
|
||||
|
||||
if(db == nil)
|
||||
error(sys->sprint("ndb file %s could not be opened", path));
|
||||
|
||||
sysname_fd := sys->open("#c/sysname", Sys->OREAD);
|
||||
if(sysname_fd == nil)
|
||||
error(sys->sprint("#c/sysname could not be read"));
|
||||
n = sys->read(sysname_fd, buf, len buf);
|
||||
if(n <= 0)
|
||||
error(sys->sprint("could not read sysname"));
|
||||
sysname = string buf[0:n];
|
||||
|
||||
if(len nodename == 0) {
|
||||
nodename = sysname;
|
||||
}
|
||||
|
||||
thiscfg: ref NodeConfig;
|
||||
nodecfgs: list of NodeConfig;
|
||||
entry: ref Dbentry;
|
||||
dbptr: ref Dbptr;
|
||||
|
||||
(entry, dbptr) = db.find(dbptr, KNAME);
|
||||
while(entry != nil) {
|
||||
nodecfg := NodeConfig.new(entry);
|
||||
|
||||
if(debug)
|
||||
sys->fprint(stderr, "config: found node %s\n", nodecfg.name);
|
||||
if(nodecfg.name == nodename) {
|
||||
thiscfg = ref nodecfg;
|
||||
}
|
||||
else
|
||||
nodecfgs = nodecfg :: nodecfgs;
|
||||
|
||||
(entry, dbptr) = db.find(dbptr, KNAME);
|
||||
}
|
||||
|
||||
if(thiscfg == nil)
|
||||
error("could not find config for nodename");
|
||||
|
||||
if(thiscfg.addr == "")
|
||||
error("node lacks an address");
|
||||
|
||||
if(thiscfg.storage == "")
|
||||
error("node lacks a storage path");
|
||||
|
||||
if(thiscfg.sysn != sysname)
|
||||
error("node and system sysname do not match");
|
||||
|
||||
return Config(
|
||||
thiscfg.name, thiscfg.sysn, thiscfg.addr, # own configuration
|
||||
thiscfg.storage, thiscfg.fswrks,
|
||||
nodecfgs); # configured nodes
|
||||
}
|
||||
|
||||
NodeConfig.new(entry: ref Dbentry): NodeConfig
|
||||
{
|
||||
name := entry.findfirst(KNAME);
|
||||
sysname := entry.findfirst(KSYSNAME);
|
||||
addr := entry.findfirst(KADDR);
|
||||
storage := entry.findfirst(KSTORAGE);
|
||||
fswrks_s := entry.findfirst(KFSWRKS);
|
||||
|
||||
fswrks := DFSWRKS;
|
||||
|
||||
if(len name == 0)
|
||||
error("entry has no name");
|
||||
if(len sysname == 0)
|
||||
error("entry has no sysname");
|
||||
if(len addr == 0)
|
||||
addr = DADDR;
|
||||
if(len fswrks_s != 0) {
|
||||
(fswrks_i, rm) := strm->toint(fswrks_s, 10);
|
||||
if(rm != "")
|
||||
error(sys->sprint("malformed fs workers count: %s", fswrks_s));
|
||||
fswrks = fswrks_i;
|
||||
}
|
||||
|
||||
return NodeConfig(
|
||||
name, sysname, addr, storage, # basic information
|
||||
fswrks); # tunable options
|
||||
}
|
||||
|
|
@ -0,0 +1,222 @@
|
|||
include "dial.m";
|
||||
dial: Dial;
|
||||
|
||||
include "security.m";
|
||||
auth: Auth;
|
||||
|
||||
include "styx.m";
|
||||
styx: Styx;
|
||||
Tmsg, Rmsg: import Styx;
|
||||
|
||||
include "styxservers.m";
|
||||
styxservers: Styxservers;
|
||||
Styxserver, Fid, Navigator,
|
||||
Navop, Enotfound, Enotdir: import styxservers;
|
||||
|
||||
# FS file index
|
||||
Qroot, Qctl, Qstats, Qmax: con iota;
|
||||
tab := array[] of {
|
||||
(Qroot, ".", Sys->DMDIR|8r555),
|
||||
(Qctl, "ctl", 8r222),
|
||||
(Qstats, "stats", 8r111),
|
||||
};
|
||||
|
||||
# create ctlfs and the appropriate listeners
|
||||
init_ctlfs(cfg: Config, keyfile: string, algs: list of string)
|
||||
{
|
||||
dial = load Dial Dial->PATH;
|
||||
auth = load Auth Auth->PATH;
|
||||
styx = load Styx Styx->PATH;
|
||||
|
||||
if(dial == nil)
|
||||
error("ctlfs: dial module not found");
|
||||
if(auth == nil)
|
||||
error("ctlfs: auth module not found");
|
||||
if(styx == nil)
|
||||
error("ctlfs: styx module not found");
|
||||
|
||||
auth->init();
|
||||
|
||||
styx->init();
|
||||
styxservers->init(styx);
|
||||
styxservers->traceset(chatty);
|
||||
|
||||
# authinfo init
|
||||
if(debug)
|
||||
sys->fprint(stderr, "ctlfs: reading authinfo");
|
||||
authinfo: ref Keyring->Authinfo;
|
||||
if (doauth) {
|
||||
if (keyfile == nil)
|
||||
keyfile = "/usr/" + user() + "/keyring/default";
|
||||
authinfo = keyring->readauthinfo(keyfile);
|
||||
if (authinfo == nil)
|
||||
error(sys->sprint("ctlfs: cannot read %s: %r", keyfile));
|
||||
}
|
||||
|
||||
# announcing
|
||||
if(debug)
|
||||
sys->fprint(stderr, "ctlfs: announcing dddbctl");
|
||||
addr := dial->netmkaddr(cfg.addr, "tcp", "dddbctl");
|
||||
c := dial->announce(addr);
|
||||
if(c == nil)
|
||||
error(sys->sprint("ctlfs: cannot listen on %s\n", addr));
|
||||
|
||||
# bootstrapping
|
||||
if(debug)
|
||||
sys->fprint
|
||||
sys->unmount(nil, "/mnt/keys");
|
||||
|
||||
navch := chan of ref Navop;
|
||||
spawn ctlfs_navigator(navch);
|
||||
|
||||
nav := Navigator.new(navch);
|
||||
(tc, srv) := Styxserver.new(fildes(0), nav, big Qroot);
|
||||
|
||||
# listener entrypoint
|
||||
listener(c, authinfo, algs);
|
||||
}
|
||||
|
||||
# dddbctl listener loop
|
||||
ctlfs_listener(c: ref Dial->Connection, authinfo: ref Keyring->Authinfo, algs: list of string)
|
||||
{
|
||||
for (;;) {
|
||||
nc := dial->listen(c);
|
||||
if (nc == nil)
|
||||
error(sys->sprint("listen failed: %r"));
|
||||
if (debug)
|
||||
sys->fprint(stderr, "ctlfs: got connection from %s\n",
|
||||
readfile(nc.dir + "/remote"));
|
||||
dfd := dial->accept(nc);
|
||||
if (dfd != nil) {
|
||||
if(nc.cfd != nil)
|
||||
sys->fprint(nc.cfd, "keepalive");
|
||||
hostname: string;
|
||||
if(passhostnames){
|
||||
hostname = readfile(nc.dir + "/remote");
|
||||
if(hostname != nil)
|
||||
hostname = hostname[0:len hostname - 1];
|
||||
}
|
||||
|
||||
spawn ctlfs_authenticator(dfd, authinfo, algs, hostname);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# authenticate a connection and set the user id.
|
||||
ctlfs_authenticator(dfd: ref Sys->FD, authinfo: ref Keyring->Authinfo,
|
||||
algs: list of string, hostname: string)
|
||||
{
|
||||
# authenticate and change user id appropriately
|
||||
(fd, err) := auth->server(algs, authinfo, dfd, 1);
|
||||
if (fd == nil) {
|
||||
if (debug)
|
||||
sys->fprint(stderr(), "ctlfs: authentication failed: %s\n", err);
|
||||
return;
|
||||
}
|
||||
if (debug)
|
||||
sys->fprint(stderr(), "ctlfs: client authenticated as %s\n", err);
|
||||
|
||||
spawn exportproc(sync, mfd, err, hostname, fd);
|
||||
}
|
||||
|
||||
ctlfs_loop()
|
||||
{
|
||||
# Primary server loop
|
||||
loop:
|
||||
while((tmsg := <-tc) != nil) {
|
||||
# Switch on operations being performed on a given Fid
|
||||
pick msg := tmsg {
|
||||
Open =>
|
||||
srv.default(msg);
|
||||
Read =>
|
||||
fid := srv.getfid(msg.fid);
|
||||
|
||||
if(fid.qtype & Sys->QTDIR) {
|
||||
# This is a directory read
|
||||
srv.default(msg);
|
||||
continue loop;
|
||||
}
|
||||
|
||||
case int fid.path {
|
||||
Qlog =>
|
||||
# A read on our log file, tell them what they've already said ?
|
||||
s := "";
|
||||
|
||||
for(l := log; l != nil; l = tl l)
|
||||
s = hd l + s;
|
||||
|
||||
srv.reply(styxservers->readstr(msg, s));
|
||||
|
||||
* =>
|
||||
srv.default(msg);
|
||||
}
|
||||
|
||||
Write =>
|
||||
fid := srv.getfid(msg.fid);
|
||||
|
||||
case int fid.path {
|
||||
Qctl =>
|
||||
# Don't care about offset
|
||||
cmd := string msg.data;
|
||||
|
||||
reply: ref Rmsg = ref Rmsg.Write(msg.tag, len msg.data);
|
||||
|
||||
case cmd {
|
||||
* =>
|
||||
# Ignore empty writes
|
||||
if(cmd != nil)
|
||||
log = cmd :: log;
|
||||
else
|
||||
reply = ref Rmsg.Error(msg.tag, "empty write!");
|
||||
}
|
||||
srv.reply(reply);
|
||||
|
||||
* =>
|
||||
srv.default(msg);
|
||||
}
|
||||
|
||||
* =>
|
||||
srv.default(msg);
|
||||
}
|
||||
}
|
||||
|
||||
exit;
|
||||
}
|
||||
|
||||
# Navigator function for moving around under /
|
||||
ctlfs_navigator(c: chan of ref Navop) {
|
||||
loop:
|
||||
for(;;) {
|
||||
navop := <-c;
|
||||
pick op := navop {
|
||||
Stat =>
|
||||
op.reply <-= (dir(int op.path), nil);
|
||||
|
||||
Walk =>
|
||||
if(op.name == "..") {
|
||||
op.reply <-= (dir(Qroot), nil);
|
||||
continue loop;
|
||||
}
|
||||
|
||||
case int op.path&16rff {
|
||||
|
||||
Qroot =>
|
||||
for(i := 1; i < Qmax; i++)
|
||||
if(tab[i].t1 == op.name) {
|
||||
op.reply <-= (dir(i), nil);
|
||||
continue loop;
|
||||
}
|
||||
|
||||
op.reply <-= (nil, Enotfound);
|
||||
* =>
|
||||
op.reply <-= (nil, Enotdir);
|
||||
}
|
||||
|
||||
Readdir =>
|
||||
for(i := 0; i < op.count && i + op.offset < (len tab) - 1; i++)
|
||||
op.reply <-= (dir(Qroot+1+i+op.offset), nil);
|
||||
|
||||
op.reply <-= (nil, nil);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,94 @@
|
|||
implement Dddb;
|
||||
|
||||
include "sys.m";
|
||||
sys: Sys;
|
||||
include "arg.m";
|
||||
include "draw.m";
|
||||
include "string.m";
|
||||
strm: String;
|
||||
|
||||
include "config.b";
|
||||
include "ctlfs.b";
|
||||
|
||||
stderr: ref Sys->FD;
|
||||
debug: int;
|
||||
|
||||
error(s: string)
|
||||
{
|
||||
sys->fprint(stderr, "dddb: %s\n", s);
|
||||
raise "dddb:error";
|
||||
}
|
||||
|
||||
Dddb: module {
|
||||
init: fn(nil: ref Draw->Context, args: list of string);
|
||||
run_fs: fn(cfg: Config);
|
||||
|
||||
Config: adt {
|
||||
name: string;
|
||||
sysn: string;
|
||||
addr: string;
|
||||
storage: string;
|
||||
fswrks: int;
|
||||
nodes: list of NodeConfig;
|
||||
|
||||
open: fn(nodename: string, path: string): Config;
|
||||
};
|
||||
|
||||
NodeConfig: adt {
|
||||
name: string;
|
||||
sysn: string;
|
||||
addr: string;
|
||||
storage: string;
|
||||
fswrks: int;
|
||||
|
||||
new: fn(entry: ref Dbentry): NodeConfig;
|
||||
};
|
||||
|
||||
};
|
||||
|
||||
init(nil: ref Draw->Context, args: list of string)
|
||||
{
|
||||
sys = load Sys Sys->PATH;
|
||||
arg := load Arg Arg->PATH;
|
||||
strm = load String String->PATH;
|
||||
|
||||
stderr = sys->fildes(2);
|
||||
cfgpath: string = "";
|
||||
|
||||
arg->init(args);
|
||||
arg->setusage(arg->progname()+ " [-d] [-c config] nodename");
|
||||
while((c := arg->opt()) != 0)
|
||||
case c {
|
||||
'd' => debug++;
|
||||
'c' =>
|
||||
cfgpath = arg->earg();
|
||||
* =>
|
||||
sys->fprint(sys->fildes(2), "bad option: -%c\n", c);
|
||||
arg->usage();
|
||||
}
|
||||
|
||||
args = arg->argv();
|
||||
|
||||
nodename := hd args;
|
||||
|
||||
if(nodename == nil) {
|
||||
sys->fprint(stderr, "dddb: no nodename supplied\n");
|
||||
arg->usage();
|
||||
}
|
||||
|
||||
if(debug)
|
||||
sys->fprint(stderr, "dddb: opening config file\n");
|
||||
cfg := Config.open(nodename, cfgpath);
|
||||
|
||||
if(debug) {
|
||||
sys->fprint(stderr, "dddb: database parms:\n");
|
||||
sys->fprint(stderr, "cfg.name: %s\n", cfg.name);
|
||||
sys->fprint(stderr, "cfg.sysn: %s\n", cfg.sysn);
|
||||
sys->fprint(stderr, "cfg.storage: %s\n", cfg.storage);
|
||||
sys->fprint(stderr, "cfg.fswrks: %d\n", cfg.fswrks);
|
||||
}
|
||||
|
||||
run_fs(cfg);
|
||||
}
|
||||
|
||||
|
||||
|
|
@ -0,0 +1,18 @@
|
|||
<../../mkconfig
|
||||
|
||||
TARG=\
|
||||
dddb.dis\
|
||||
|
||||
MODULES=\
|
||||
|
||||
SYSMODULES=\
|
||||
arg.m\
|
||||
dial.m\
|
||||
draw.m\
|
||||
sys.m\
|
||||
styx.m\
|
||||
styxservers.m\
|
||||
|
||||
DISBIN=$home/dis
|
||||
|
||||
<$ROOT/mkfiles/mkdis
|
||||
|
|
@ -0,0 +1,11 @@
|
|||
<../../mkconfig
|
||||
|
||||
TARG=\
|
||||
|
||||
MODULES=\
|
||||
|
||||
SYSMODULES=\
|
||||
|
||||
DISBIN=$ROOT/dis/lib
|
||||
|
||||
<$ROOT/mkfiles/mkdis
|
||||
|
|
@ -0,0 +1,6 @@
|
|||
<../mkconfig
|
||||
|
||||
DIRS=\
|
||||
cmd\
|
||||
|
||||
<$ROOT/mkfiles/mksubdirs
|
||||
|
|
@ -0,0 +1,6 @@
|
|||
<$ROOT/mkconfig
|
||||
|
||||
# Comment the below lines for hosted builds
|
||||
SYSHOST=Inferno
|
||||
SYSTARG=Inferno
|
||||
ROOT=/
|
||||
Loading…
Reference in New Issue