#!/usr/bin/env qore
# -*- mode: qore; indent-tabs-mode: nil -*-

# @file qdp command-line interface for the DataProvider module

/*  Copyright 2019 - 2025 Qore Technologies, s.r.o.

    Permission is hereby granted, free of charge, to any person obtaining a
    copy of this software and associated documentation files (the "Software"),
    to deal in the Software without restriction, including without limitation
    the rights to use, copy, modify, merge, publish, distribute, sublicense,
    and/or sell copies of the Software, and to permit persons to whom the
    Software is furnished to do so, subject to the following conditions:

    The above copyright notice and this permission notice shall be included in
    all copies or substantial portions of the Software.

    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
    DEALINGS IN THE SOFTWARE.
*/

%new-style
%enable-all-warnings
%require-types
%strict-args

%requires qore >= 1.0

%requires ConnectionProvider
%requires DatasourceProvider
%requires DbDataProvider
%requires Util
%requires Logger
# for webhooks
%requires HttpServer
%requires RestHandler
# to create the data provider index
%requires ProviderIndex

%try-module linenoise
%define NoLinenoise
%endtry

%exec-class QdpCmd

class QdpCmd {
    public {
        static HttpServer http;

        static string cmd;
        static string app;
        static string action;

        static bool ix;

        #! port for the HTTP server
        static int http_port = 0;

        # Global mutex for webhook handlers
        static Mutex wh_lock();

        # Global map for webhook handlers: webhook -> handler
        static hash<string, WebhookHandler> whm;

        # console logger
        LoggerInterface logger;

        #! program options
        const Opts = {
            "bulk": "b,bulk",
            "createindex": "I,create-index",
            "host": "H,host=s",
            "listapps": "a,list-apps",
            "listconnections": "c,list-connections",
            "listfactories": "f,list-factories",
            "listtypes": "t,list-types",
            "modules": "m,module=s@",
            "options": "o,options=s",
            "required": "R,show-required",
            "servers": "S,servers",
            "subtype": "s,subtype=s",
            "verbose": "v,verbose:i+",
            "exp": "x,expressions",
            "help": "h,help",
        };

        #! commands
        const Cmds = {
            "action-info": \QdpCmd::actionInfo(),
            "ai": \QdpCmd::actionInfo(),
            "ainfo": \QdpCmd::actionInfo(),
            "create": \QdpCmd::create(),
            "create-record": \QdpCmd::getCreateRecord(),
            "crecord": \QdpCmd::getCreateRecord(),
            "delete": \QdpCmd::del(),
            "dorequest": \QdpCmd::doRequest(),
            "errors": \QdpCmd::errors(),
            "event": \QdpCmd::event(),
            "example-error": \QdpCmd::exampleError(),
            "example-event": \QdpCmd::exampleEvent(),
            "example-record": \QdpCmd::exampleRecord(),
            "example-request": \QdpCmd::exampleRequest(),
            "example-response": \QdpCmd::exampleResponse(),
            "exerror": \QdpCmd::exampleError(),
            "exevent": \QdpCmd::exampleEvent(),
            "exrecord": \QdpCmd::exampleRecord(),
            "exrequest": \QdpCmd::exampleRequest(),
            "exresponse": \QdpCmd::exampleResponse(),
            "fadd": \QdpCmd::fieldAdd(),
            "fdelete": \QdpCmd::fieldDelete(),
            "field-add": \QdpCmd::fieldAdd(),
            "field-del": \QdpCmd::fieldDelete(),
            "field-update": \QdpCmd::fieldUpdate(),
            "fupdate": \QdpCmd::fieldUpdate(),
            "info": \QdpCmd::getInfo(),
            "interactive": \QdpCmd::interactive(),
            "ix": \QdpCmd::interactive(),
            "ldetails": \QdpCmd::listChildDetails(),
            "levents": \QdpCmd::listEvents(),
            "list": \QdpCmd::listChildren(),
            "list-details": \QdpCmd::listChildDetails(),
            "list-events": \QdpCmd::listEvents(),
            "listen": \QdpCmd::listen(),
            "lmessages": \QdpCmd::listMessages(),
            "lmsgs": \QdpCmd::listMessages(),
            "message": \QdpCmd::message(),
            "msg": \QdpCmd::message(),
            "pcreate": \QdpCmd::providerCreate(),
            "pdelete": \QdpCmd::providerDelete(),
            "provider-create": \QdpCmd::providerCreate(),
            "provider-delete": \QdpCmd::providerDelete(),
            "record": \QdpCmd::getRecord(),
            "reply": \QdpCmd::response(),
            "request": \QdpCmd::request(),
            "response": \QdpCmd::response(),
            "rsearch": \QdpCmd::doRequestSearch(),
            "search": \QdpCmd::search(),
            "send-message": \QdpCmd::sendMessage(),
            "show-providers": \QdpCmd::showProviders(),
            "smessage": \QdpCmd::sendMessage(),
            "smsg": \QdpCmd::sendMessage(),
            "sp": \QdpCmd::showProviders(),
            "update": \QdpCmd::update(),
            "update-record": \QdpCmd::getUpdateRecord(),
            "urecord": \QdpCmd::getUpdateRecord(),
            "upsert": \QdpCmd::upsert(),
        };

        #! Commands that don't need a data provider
        const NoProviderCmds = (
            "action-info": True,
            "ai": True,
            "ainfo": True,
        );

        #! Disambiguation for common commands with the same initial letters
        const DisambiguationMap = {
            "search": "supports_read",
            "send-message": "supports_messages",
            "smessage": "supports_messages",
            "smsg": "supports_messages",

            "record": "has_record",
            "reply": "supports_request",
            "request": "supports_request",
            "response": "supports_request",
            "rsearch": "supports_request",
        };

        #! valid field keys
        const FieldKeys = (
            "type",
            "desc",
            "default_value",
            "opts",
        );
    }

    constructor() {
        # must be called before any data provider operations are performed to allow for automatic configuration from
        # environment variables by supported data provider modules (ex: SalesforceRestDataProvider)
        DataProvider::setAutoConfig();

        GetOpt g(Opts);
        our hash<auto> opts = g.parse3(\ARGV);
        if (opts.help) {
            usage();
        }

        # load any modules
        map load_module($1), opts.modules;

        logger = getLogger();
        DataProvider::setGlobalLogger(logger);
        ProviderIndex::setGlobalLogger(logger);

        if (opts.listconnections) {
            listConnections();
        }
        if (opts.listfactories) {
            listFactories();
        }
        if (opts.listtypes) {
            listTypes();
        }
        if (opts.listapps) {
            listApps();
        }
        if (opts.createindex) {
            createIndex();
        }
        if (!ARGV[0]) {
            usage();
        }

        on_exit {
            DataProvider::shutdown();
        }

        DataProviderDataContextHelper dch;
        if (opts.options.val()) {
            auto v = parse_to_qore_value(opts.options);
            if (v.typeCode() != NT_HASH) {
                error("options have type %y; expecting \"hash\"", v.fullType());
            }
            dch = new DataProviderDataContextHelper(v);
        }

        logger = getLogger();
        DataProvider::setGlobalLogger(logger);

        if (opts.servers) {
            DataProvider::addOptions(DPO_EnableServers);
        }

        on_error rethrow $1.err, sprintf("%s%s", $1.desc,
            $1.arg instanceof object && $1.arg.hasCallableMethod("getCause")
                ? ": " + $1.arg.getCause().toString()
                : NOTHING);

        AbstractDataProvider provider;
        DataProviderContextHelper pctx;
        try {
            # first extract any data provider options
            string name = shift ARGV;
            # get expression in curly brackets, if any, respecting balanced brackets
            *string opts = (name =~ x/({(?:(?>[^{}]*)|(?0))*})/)[0];
            if (opts) {
                name = replace(name, opts, "");
            } else {
                # remove empty options; the above regex does not catch them :(
                name =~ s/{}//;
            }
            if (name =~ /^@/) {
                splice name, 0, 1;
                # ignore any trailing "/"
                name =~ s/\/+$//;
                list<string> path = name.split("/");
                if (path.size() == 1) {
                    showActions(name);
                }
                app = path[0];
                action = path[1];
                provider = getDataProviderForAppAction(app, action, opts);
            } else {
                list<string> path = name.split("/");
                provider = getDataProvider((shift path) + opts);
                map provider = provider.getChildProviderEx($1), path;
            }
            provider.setLogger(logger);
            pctx = new DataProviderContextHelper(provider);
        } catch (hash<ExceptionInfo> ex) {
            if (!NoProviderCmds{ARGV[0]}) {
                rethrow;
            }
        }

        # Set up fake webhook interface for the TypeScriptActionInterface module if possible
        if (get_module_hash().TypeScriptActionInterface) {
            # make sure symbols are loaded in this Program
            load_module("TypeScriptActionInterface");
            Class cls = Class::forName("TypeScriptActionInterface::TypeScriptActionInterface");
            hash<MethodAccessInfo> info = cls.findStaticMethod("registerWebhookCreateDestroyApis");
            StaticMethodVariant v = info.method.getVariants()[0];
            v.call(\QdpCmd::getWebhook(), \QdpCmd::deleteWebhook());
        }

        on_exit if (http) {
            http.stop();
            delete http;
        }

        cmd = shift ARGV ?? "list";
        *code qdp_action = Cmds{cmd};
        if (!qdp_action) {
            # see if an abbreviation matches
            hash<string, bool> match = map {$1: True}, keys Cmds, $1.equalPartial(cmd);
            # try to disambiguate
            if (provider && (match.size() > 1)) {
                # only consider commands that are applicable for this data provider
                # AbstractDataProvider::getSummaryInfo() is generally faster than getInfo()
                hash<DataProviderSummaryInfo> info = provider.getSummaryInfo();
                match = map {$1: True}, keys match, !exists DisambiguationMap{$1}
                    || (info{DisambiguationMap{$1}} === True)
                    || (info{DisambiguationMap{$1}}.typeCode() == NT_STRING && info{DisambiguationMap{$1}} != "NONE");
            }
            if (match.size() == 1) {
                qdp_action = Cmds{match.firstKey()};
            } else if (match) {
                # check if all matches have the same action
                {
                    foreach string c in (keys match) {
                        if (!qdp_action) {
                            qdp_action = Cmds{c};
                        } else if (Cmds{c} != qdp_action) {
                            break;
                        }
                    }
                }
                if (!qdp_action) {
                    error("unknown action %y; matches %y; please provide additional character(s) to ensure a unique "
                        "match", cmd, keys match);
                }
            } else {
                error("unknown action %y; known actions: %y", cmd, keys Cmds);
            }
        }
        try {
            qdp_action(provider);
        } catch (hash<ExceptionInfo> ex) {
            if (ex.err == "INVALID-OPERATION") {
                error("%s: %s", ex.err, ex.desc);
            } else {
                rethrow;
            }
        }
    }

    hash<auto> getWebhook(hash<auto> h) {
        if (h.method.typeCode() != NT_STRING) {
            throw "WEBHOOK-ERROR", sprintf("\"method\" must be a string when registering a webhook; got type %y (%y)",
                h.method.fullType(), h.method);
        }
        if (!h.callback.callp()) {
            throw "WEBHOOK-ERROR", sprintf("\"callback\" must be a callable value when registering a webhook; got "
                "type %y (%y)", h.callback.fullType(), h.method);
        }
        # auth is validated but ignored
        if (exists h.auth && (h.auth.typeCode() != NT_INT)) {
            throw "WEBHOOK-ERROR", sprintf("\"auth\" must be a valid auth code if present when registering a "
                "webhook; got type %y (%y)", h.auth.fullType(), h.auth);
        }
        # perms is validated but ignored
        if (exists h.perms && h.perms.typeCode() != NT_LIST) {
            throw "WEBHOOK-ERROR", sprintf("\"perms\" must be a list of strings if present when registering a "
                "webhook; got type %y (%y)", h.perms.fullType(), h.perms);
        }

        setupHttpServer();

        string path = get_random_string();

        WebhookHandler handler(h.method, path, h.callback);
        hash<HttpHandlerConfigInfo> hinfo = <HttpHandlerConfigInfo>{
            "path": path,
            "handler": handler,
        };

        AutoLock al(wh_lock);
        http.setDynamicHandler(path, hinfo);
        whm{path} = handler;

        string http_host = opts.host ?? "localhost";
        if (*softint p = (http_host =~ x/:([0-9]+)$/)[0]) {
            http_host =~ s/:.*$//;
            http_port = p;
        }

        return {
            "url": sprintf("http://%s:%d/webhooks/%s", http_host, http_port, path),
        };
    }

    deleteWebhook(hash<auto> info) {
        if (http) {
            string path = (info.url =~ x/\/([^\/]+)$/)[0];
            http.removeDynamicHandler(path, True);
        }
    }

    synchronized setupHttpServer() {
        if (http) {
            return;
        }
        hash<HttpServerOptionInfo> http_opts = <HttpServerOptionInfo>{
            "logger": getLogger("http"),
            "debug": True,
        };

        http = new HttpServer(http_opts);
        http_port = http.addListener(<HttpListenerOptionInfo>{"service": 0}).port;
    }

    private Logger getLogger(string name = "console") {
        LoggerLevel level;
        if (opts.verbose > 1) {
            level = LoggerLevel::getLevelDebug();
        } else if (opts.verbose) {
            level = LoggerLevel::getLevelInfo();
        } else {
            level = LoggerLevel::getLevelError();
        }
        Logger logger(name, level);
        logger.addAppender(new ConsoleAppender());
        return logger;
    }

    static exampleRequest(AbstractDataProvider provider) {
        printf("%N\n", provider.getExampleRequestData());
    }

    static exampleResponse(AbstractDataProvider provider) {
        printf("%N\n", provider.getExampleResponseData());
    }

    static exampleRecord(AbstractDataProvider provider) {
        printf("%N\n", provider.getExampleRecordData());
    }

    static listEvents(AbstractDataProvider provider) {
        hash<string, hash<DataProviderMessageInfo>> events = provider.getEventTypes();
        map printf(" - %s\n", $1), keys events;
    }

    static event(AbstractDataProvider provider) {
        *string event = QdpCmd::getString("event");
        *hash<auto> req;
        hash<DataProviderMessageInfo> event_info;
        if (event.val() && event =~ /=/) {
            req = getHashIntern(remove event, "request");
        }
        if (event.val()) {
            req = QdpCmd::getHash("request");
            event_info = provider.getEventInfoWithData(event, req);
        }
        if (!event.val()) {
            hash<string, hash<DataProviderMessageInfo>> event_types = provider.getEventTypes();
            if (event_types.size() == 1) {
                event = event_types.firstKey();
                if (req) {
                    event_info = provider.getEventInfoWithData(event, req);
                } else {
                    event_info = event_types.firstValue();
                }
            } else {
                QdpCmd::error("Missing required event argument; valid events: %y", keys event_types);
            }
        }
        DataProviderDataContextHelper dch(req);
        QdpCmd::showType(provider, event, event_info);
    }

    static exampleEvent(AbstractDataProvider provider) {
        *string event = QdpCmd::getString("event");
        if (!exists event) {
            *list<string> l = keys provider.getEventTypes();
            if (l.size() == 1) {
                event = l[0];
            } else {
                QdpCmd::error("Missing required event argument; valid events: %y", l);
            }
        }
        printf("%N\n", provider.getExampleEventData(event));
    }

    static errors(AbstractDataProvider provider) {
        *hash<string, AbstractDataProviderType> errs = provider.getErrorResponseTypes();
        *string err = QdpCmd::getString("error");
        if (err) {
            *AbstractDataProviderType type = errs{err};
            if (!type) {
                QdpCmd::error("unknown error code %y; known error codes: %y", err, keys errs);
            }
            QdpCmd::showType(provider, type);
            return;
        }

        if (!opts.verbose) {
            printf("%y\n", keys errs);
        } else {
            map (printf("%s:\n", $1.key), QdpCmd::showType(provider, $1.value, "  ")), errs.pairIterator();
        }
    }

    static exampleError(AbstractDataProvider provider) {
        string error = QdpCmd::getString("error", True);
        printf("%N\n", provider.getExampleErrorResponseData(error));
    }

    static AbstractDataField getField(string name, auto f) {
        if (f.typeCode() != NT_HASH) {
            QdpCmd::error("field %y value must be type \"hash\"; got type %y instead", name, f.type());
        }
        if (*hash<auto> err = (f - FieldKeys)) {
            QdpCmd::error("field %y has unknown keys: %y; known keys: %y", name, keys err, FieldKeys);
        }
        return new QoreDataField(name, f.desc, AbstractDataProviderType::get(f.type ?? "string", f.opts),
            f.default_value);
    }

    static providerCreate(AbstractDataProvider provider) {
        string name = QdpCmd::getString("provider-create 'name'", True);
        hash<auto> desc = QdpCmd::getHash("provider-create description", True);
        *hash<auto> opts = QdpCmd::getHash("provider-create options");

        hash<string, AbstractDataField> fields = map {$1.key: QdpCmd::getField($1.key, $1.value)}, desc.pairIterator();
        provider.createChildProvider(name, fields, opts);
        printf("created provider %s/%s\n", provider.getName(), name);
    }

    static providerDelete(AbstractDataProvider provider) {
        string name = QdpCmd::getString("provider-delete 'name'", True);
        provider.deleteChildProvider(name);
        printf("deleted provider %s/%s\n", provider.getName(), name);
    }

    static fieldAdd(AbstractDataProvider provider) {
        string name = QdpCmd::getString("field-add 'name'", True);
        hash<auto> desc = QdpCmd::getHash("field-add description", True);
        *hash<auto> opts = QdpCmd::getHash("field-add options");
        AbstractDataField field = QdpCmd::getField(name, desc);
        provider.addField(field, opts);
        printf("added field %y to provider %y\n", name, provider.getName());
    }

    static fieldUpdate(AbstractDataProvider provider) {
        string old_name = QdpCmd::getString("field-update 'old name'", True);
        string new_name = QdpCmd::getString("field-update 'new name'", True);
        *hash<auto> desc = QdpCmd::getHash("field-update description", True);
        *hash<auto> opts = QdpCmd::getHash("field-update options");
        AbstractDataField field;
        if (desc) {
            field = QdpCmd::getField(new_name, desc);
        } else {
            *hash<string, AbstractDataField> fields = provider.getRecordType();
            if (!fields{old_name}) {
                QdpCmd::error("provider %y has no field %y to update", provider.getName(), old_name);
            }
            field = fields{old_name};
        }
        provider.updateField(old_name, field, opts);
        if (old_name == new_name) {
            printf("updated field %y in provider %y\n", old_name, provider.getName());
        } else {
            printf("renamed field %y -> %y in provider %y\n", old_name, new_name, provider.getName());
        }
    }

    static fieldDelete(AbstractDataProvider provider) {
        string name = QdpCmd::getString("field-delete 'name'", True);
        provider.deleteField(name);
        printf("deleted field %y from provider %s/%s\n", name, provider.getName());
    }

    static getInfo(AbstractDataProvider provider) {
        printf("%N\n", provider.getInfo());
    }

    static listChildren(AbstractDataProvider provider) {
        *list<string> children = provider.getChildProviderNames();
        if (children) {
            printf("%y\n", children);
        } else {
            printf(provider.getName() + " has no children\n");
        }
    }

    static listChildDetails(AbstractDataProvider provider) {
        *list<hash<DataProviderSummaryInfo>> children = provider.getChildProviderSummaryInfo();
        if (children) {
            code get_code = string sub (hash<auto> h) {
                if (h.has_record) {
                    return "REC";
                }
                if (h.supports_request) {
                    return "API";
                }
                if (h.supports_observable) {
                    return "EVT";
                }
                if (h.supports_messages != MSG_None) {
                    return "MSG";
                }
                if (h.children_can_support_records) {
                    return "REC";
                }
                if (h.children_can_support_apis) {
                    return "API";
                }
                if (h.children_can_support_observers) {
                    return "EVT";
                }
                if (h.children_can_support_messages) {
                    return "MSG";
                }
                return "---";
            };
            map printf("- %s: (%s) %s\n", $1.name, get_code($1), QdpCmd::getDesc(10, $1, "name")), children;
        } else {
            printf(provider.getName() + " has no children\n");
        }
    }

    static string getDesc(int offset, hash<auto> h) {
        int len = TermIOS::getWindowSize().columns - offset;
        map len -= h{$1}.length(), argv;
        string desc = h.desc ?? "";
        desc =~ s/\n/ /g;
        desc =~ s/ +/ /g;
        if (len < 0) {
            return "";
        }
        if (desc.length() > len) {
            if (len <= 3) {
                splice desc, len;
            } else {
                splice desc, len - 3;
                desc += "...";
            }
        }
        return desc;
    }

    static interactive(AbstractDataProvider provider) {
        provider.checkObservable();
        {
            on_error rethrow $1.err, sprintf("%s (try using \"listen\" instead)", $1.desc);
            provider.checkMessages();
        }
        hash<DataProviderInfo> info = provider.getInfo();
        MyObserver observer(info);
        Observable observable = cast<Observable>(provider);

        ix = True;

        # print a message
        stdout.printf("listening to events from %y; enter <message id>: <message body> to send a message, 'help' for "
            "help\n", provider.getName());
        observable.registerObserver(observer);

        if (observable instanceof DataProvider::DelayedObservable) {
            cast<DataProvider::DelayedObservable>(observable).observersReady();
        }

        InputHelper input_helper(provider);
        while (True) {
            string input = input_helper.get();
            #printf("input: %y\n", input);
            bool quit;
            switch (input) {
                case "exit":
                case "quit":
                    quit = True;
                    break;
            }
            if (quit) {
                break;
            }
            # check for msg ID
            (*string msgid, *string msg) = (input =~ x/([^:]+):(.*)/);
            if (!msgid) {
                stdout.printf("cannot parse input %y; enter <message id>: <message body> to send a message, "
                    "'help' for help\n", input);
                continue;
            }

            auto v;
            try {
                v = parse_to_qore_value(msg);
            } catch (hash<ExceptionInfo> ex) {
                stderr.printf("%s: %s; client-side error; the message was not sent\n", ex.err, ex.desc);
                continue;
            }
            printf("> sending %y -> %y\n", msgid, v);
            try {
                provider.sendMessage(msgid, v);
            } catch (hash<ExceptionInfo> ex) {
                stderr.printf("%s: %s\n", ex.err, ex.desc);
            }
        }
        ix = False;
        printf("stopped\n");
    }

    static listMessages(AbstractDataProvider provider) {
        hash<string, hash<DataProviderMessageInfo>> messages = provider.getMessageTypes();
        map printf(" - %s\n", $1), keys messages;
    }

    static message(AbstractDataProvider provider) {
        *string message = QdpCmd::getString("message");
        if (!exists message) {
            QdpCmd::error("missing required message argument; valid messages: %y", keys provider.getMessageTypes());
        }
        QdpCmd::showType(provider, message, provider.getMessageInfo(message));
    }

    static listen(AbstractDataProvider provider) {
        provider.checkObservable();
        hash<DataProviderInfo> info = provider.getInfo();
        MyObserver observer(info);
        Observable observable = cast<Observable>(provider);

        # print a message
        stdout.printf("listening to events from %y; press any key to stop listening...\n", provider.getName());
        {
            Term t();

            observable.registerObserver(observer);

            if (observable instanceof DataProvider::DelayedObservable) {
                cast<DataProvider::DelayedObservable>(observable).observersReady();
            }

            while (observable.isActive()) {
                if (stdin.isDataAvailable(250ms)) {
                    stdin.readBinary(1);
                    break;
                }
            }
        }
        bool active = observable.isActive();
        delete provider;
        if (active) {
            printf("listening stopped on user request\n");
        } else {
            printf("listening stopped due to an error\n");
        }
    }

    static getRecord(AbstractDataProvider provider) {
        *hash<auto> search_options = QdpCmd::getHash("search options");
        *hash<string, AbstractDataField> rec = provider.getRecordType(search_options);
        if (opts.verbose > 1) {
            printf("%N\n", (map {$1.key: $1.value.getInfo()}, rec.pairIterator()));
        } else {
            QdpCmd::showRecord(provider, rec);
        }
    }

    static getCreateRecord(AbstractDataProvider provider) {
        *hash<auto> create_options = QdpCmd::getHash("create options");
        *hash<string, AbstractDataField> rec = provider.getCreateRecordType(create_options);
        if (opts.verbose > 1) {
            printf("%N\n", (map {$1.key: $1.value.getInfo()}, rec.pairIterator()));
        } else {
            QdpCmd::showRecord(provider, rec);
        }
    }

    static getUpdateRecord(AbstractDataProvider provider) {
        *hash<auto> search_options = QdpCmd::getHash("search options");
        *hash<string, AbstractDataField> rec = provider.getUpdateRecordType(search_options);
        if (opts.verbose > 1) {
            printf("%N\n", (map {$1.key: $1.value.getInfo()}, rec.pairIterator()));
        } else {
            QdpCmd::showRecord(provider, rec);
        }
    }

    static getAllowedValues(reference<string> info, *list<auto> l) {
        if (info) {
            info += " ";
        }
        if (opts.verbose && l[0].typeCode() == NT_HASH) {
            if (l[0].value != l[0].display_name) {
                info += sprintf("allowed_values: %y", (map sprintf("%s (%s)", $1.display_name, $1.value), l));
            } else {
                info += sprintf("allowed_values: %y", (map $1.value, l));
            }
        } else {
            info += sprintf("allowed_values: %y", (map $1.value ?? $1, l));
        }
    }

    static showRecord(*AbstractDataProvider prov, *hash<string, AbstractDataField> rec,
            string offset = "", auto dctxt) {
        hash<string, bool> rmap;
        showRecord(prov, \rmap, rec, offset, dctxt);
    }

    static showRecord(*AbstractDataProvider prov, reference<hash<string, bool>> rmap, *hash<string, AbstractDataField> rec,
            string offset = "", auto dctxt, *bool unsatisfied_deps) {
        foreach hash<auto> i in (rec.pairIterator()) {
            *bool local_unsatisfied_deps = unsatisfied_deps;
            auto dv = i.value.getDefaultValue();
            if (prov && !exists dv && !unsatisfied_deps) {
                if ((*string default_ref_data = i.value.getAttributes().default_ref_data).val()) {
                    dv = prov.getReferenceDataValueSafe(default_ref_data, DataProviderDataContextHelper::getHash());
                } else if (!offset.val() && prov.getSupportedReferenceDataValues(){i.key}) {
                    dv = prov.getReferenceDataValueSafeEx(i.key, DataProviderDataContextHelper::getHash());
                }
            }
            if (opts.required && (!i.value.isMandatory() || exists dv)) {
                continue;
            }
            AbstractDataProviderType type = i.value.getType();
            *hash<auto> attr = i.value.getAttributes();
            # do not request dependent values if dependencies are not satisfied
            if (!local_unsatisfied_deps && attr.depends_on && (dctxt{attr.depends_on}.size() != attr.depends_on.size())) {
                local_unsatisfied_deps = True;
            }
            showField(prov, \rmap, i, dv, type, offset, dctxt, local_unsatisfied_deps);
            *hash<string, AbstractDataField> fields = type.getFields();
            if (fields) {
                string u = type.uniqueHash();
                if (!rmap{u}) {
                    rmap{u} = True;
                    QdpCmd::showRecord(prov, \rmap, fields, offset + "  ", dctxt{i.key}, local_unsatisfied_deps);
                }
            }
        }
    }

    static showField(*AbstractDataProvider prov, reference<hash<string, bool>> rmap, hash<auto> i, auto dv,
            AbstractDataProviderType type, string offset, auto dctxt, *bool unsatisfied_deps) {
        if (opts.verbose) {
            string txt = sprintf("%s%s %s", offset, type.getName(), i.key);
            string info;
            if (*string desc = i.value.getShortDescription()) {
                info = sprintf("desc: %y", QdpCmd::getDesc(0, {"desc": desc}));
            }
            if (exists dv) {
                if (info) {
                    info += " ";
                }
                info += sprintf("default_value: %y", dv);
            }

            if (*list<auto> l = i.value.getAllowedValues()) {
                QdpCmd::getAllowedValues(\info, l);
            } else if (prov && !unsatisfied_deps && (*string ref_data = i.value.getAttributes().ref_data).val()) {
                l = prov.getReferenceDataSafe(ref_data, DataProviderDataContextHelper::getHash());
                QdpCmd::getAllowedValues(\info, l);
            } else if (prov && !unsatisfied_deps && !offset.val() && prov.getSupportedReferenceData(){i.key}
                && (l = prov.getReferenceDataSafeEx(i.key, DataProviderDataContextHelper::getHash()))) {
                QdpCmd::getAllowedValues(\info, l);
            } else if (l = i.value.getElementAllowedValues()) {
                QdpCmd::getAllowedValues(\info, l);
            } else if (prov && !unsatisfied_deps && (ref_data = i.value.getAttributes().element_ref_data).val()) {
                l = prov.getElementReferenceDataSafe(ref_data, DataProviderDataContextHelper::getHash());
                QdpCmd::getAllowedValues(\info, l);
            } else if (prov && !unsatisfied_deps && !offset.val() && prov.getSupportedElementReferenceData(){i.key}
                && (l = prov.getElementReferenceDataSafeEx(i.key, DataProviderDataContextHelper::getHash()))) {
                QdpCmd::getAllowedValues(\info, l);
            }

            if (info) {
                txt += " (" + info + ")";
            }
            print(txt);
        } else {
            string info;
            if (*list<auto> l = i.value.getAllowedValues()) {
                info += sprintf(" (one of: %y)", (map $1.value ?? $1, l));
            } else if (prov && !unsatisfied_deps && (*string ref_data = i.value.getAttributes().ref_data).val()) {
                l = prov.getReferenceDataSafe(ref_data, DataProviderDataContextHelper::getHash());
                if (l) {
                    info += sprintf(" (one of: %y)", (map $1.value ?? $1, l));
                }
            } else if (l = i.value.getElementAllowedValues()) {
                info += sprintf(" (elements: one of: %y)", (map $1.value ?? $1, l));
            } else if (prov && !unsatisfied_deps && (ref_data = i.value.getAttributes().element_ref_data).val()) {
                l = prov.getElementReferenceDataSafe(ref_data, DataProviderDataContextHelper::getHash());
                if (l) {
                    info += sprintf(" (elements: one of: %y)", (map $1.value ?? $1, l));
                }
            }
            printf("%s%s %s%s", offset, type.getName(), i.key, info);
        }
        if (type.getBaseTypeCode() == NT_LIST && (*AbstractDataProviderType element_type = type.getElementType())
            && (*hash<string, AbstractDataField> element_rec = element_type.getFields())) {
            string u = type.uniqueHash();
            if (!rmap{u}) {
                rmap{u} = True;
                print(": elements ->\n");
                QdpCmd::showRecord(prov, \rmap, element_rec, offset + "  ", dctxt{i.key}, unsatisfied_deps);
            } else {
                print(": (type already described)\n");
            }
        } else {
            print("\n");
        }
    }

    static search(AbstractDataProvider provider) {
        *hash<auto> where_cond = QdpCmd::getSearchExpression("search");
        *hash<auto> search_options = QdpCmd::getHash("search options");
        AbstractDataProviderRecordIterator i;
        if (opts.bulk) {
            i = provider.searchRecordsBulk(NOTHING, where_cond, search_options).getRecordIterator();
        } else {
            i = provider.searchRecords(where_cond, search_options);
        }
        map printf("%y\n", $1), i;
    }

    static sendMessage(AbstractDataProvider provider) {
        string message_id = QdpCmd::getString("message_id", True);
        auto val = QdpCmd::getAny("data");
        *hash<auto> send_options = QdpCmd::getHash("send options");

        if (provider instanceof DataProvider::DelayedObservable) {
            cast<DataProvider::DelayedObservable>(provider).observersReady();
        }

        provider.sendMessage(message_id, val, send_options);
    }

    static showProviders(AbstractDataProvider provider) {
        QdpCmd::showProviders("/", provider);
    }

    static showProviders(string path, AbstractDataProvider provider) {
        hash<DataProviderInfo> info = provider.getInfo();
        string type;
        if (info.supports_request) {
            type += "A";
        } else {
            type += "-";
        }
        if (info.supports_observable) {
            type += "E";
        } else {
            type += "-";
        }
        if (info.supports_messages != MSG_None) {
            type += "M";
        } else {
            type += "-";
        }
        if (info.has_record && (info.supports_read || info.supports_create || info.supports_update
            || info.supports_upsert)) {
            type += "R";
        } else {
            type += "-";
        }
        if (type != "----") {
            string desc = sprintf("(%s) %s", type, path);
            if (opts.verbose) {
                desc += sprintf(": %s", provider.getShortDesc());
            }
            printf("- %s\n", desc);
        }
        if (path !~ /\/$/) {
            path += "/";
        }
        *list<string> l = QdpCmd::time(sprintf("get provider names: %y", provider.getName()),
            \provider.getChildProviderNames());
        foreach string child in (l) {
            AbstractDataProvider cp = QdpCmd::time(sprintf("get child %y from %y (%s)", child, provider.getName(),
                provider.className()),
                \provider.getChildProviderEx(), child);
            QdpCmd::showProviders(path + child, cp);
        }

        #map QdpCmd::showProviders(path + $1, provider.getChildProviderEx($1)), provider.getChildProviderNames();
    }

    static auto time(string label, code call, ...) {
        date start = now_us();
        auto rv = call_function_args(call, argv);
        date d = now_us() - start;
        if (d > 500ms) {
            printf("CALL %y: %y\n", label, d);
        }
        return rv;
    }

    static actionInfo(*AbstractDataProvider provider) {
        if (!exists app) {
            QdpCmd::error("Missing app and action for action info; use:\n    %s @<app>/<action> %s",
                get_script_name(), cmd);
        }
        hash<DataProviderActionInfo> action = DataProviderActionCatalog::getAppActionEx(app, QdpCmd::action);
        printf("@%s/%s (%s%s): %s\n", app, QdpCmd::action, ActionNameMap{action.action_code},
            (action.action_val.val() ? " \"" + action.action_val + "\"" : ""), action.short_desc);
        action = DataProviderActionCatalog::checkAppActionOptions(action.app, action.action);
        *hash<auto> req = getHashIntern(shift ARGV, "options");
        *hash<string, hash<ActionOptionInfo>> opts = provider
            ? provider.tryGetActionOptionsWithData(action, req)
            : action.options;
        if (opts) {
            printf("Options:\n");
            foreach hash<auto> i in (opts.pairIterator()) {
                string tn = i.value.type.getName();
                # get allowed values or element allowed values
                string info;
                if (*list<auto> l = i.value.allowed_values) {
                    info += sprintf(", one of: %y", (map $1.value ?? $1, l));
                } else if (provider && (*string ref_data = i.value.ref_data).val()) {
                    l = provider.getReferenceDataSafe(ref_data, DataProviderDataContextHelper::getHash());
                    if (l) {
                        info += sprintf(", one of: %y", (map $1.value ?? $1, l));
                    }
                } else if (l = i.value.element_allowed_values) {
                    info += sprintf(", elements: one of: %y", (map $1.value ?? $1, l));
                } else if (provider && (ref_data = i.value.element_ref_data).val()) {
                    l = provider.getElementReferenceDataSafe(ref_data, DataProviderDataContextHelper::getHash());
                    if (l) {
                        info += sprintf(", elements: one of: %y", (map $1.value ?? $1, l));
                    }
                }
                printf(" - %s (%s%s%s): %s\n", i.key, i.value.required || tn =~ /^\*/ ? "" : "*", tn, info,
                    i.value.short_desc);
            }
        } else {
            printf("Options: none\n");
        }
        if (action.output_type) {
            printf("Output:\n");
            QdpCmd::showType(provider, action.output_type, "  ");
        }
    }

    static update(AbstractDataProvider provider) {
        hash<auto> set = QdpCmd::getHash("update 'set'", True);
        *hash<auto> where_cond = QdpCmd::getSearchExpression("update 'where'", True);
        int rec_count = provider.updateRecords(set, where_cond);
        printf("%d record%s updated\n", rec_count, rec_count == 1 ? "" : "s");
    }

    static create(AbstractDataProvider provider) {
        *hash<auto> rec = QdpCmd::getHash("create");
        if (!exists rec) {
            QdpCmd::error("missing argument to 'create' action; a hash is required");
        }
        *hash<auto> rv = provider.createRecord(rec);
        if (opts.verbose) {
            printf("new record: %y\n", rv);
        }
        printf("record successfully created\n");
    }

    static upsert(AbstractDataProvider provider) {
        *hash<auto> rec = QdpCmd::getHash("upsert");
        *hash<auto> upsert_options = QdpCmd::getHash("upsert options");
        string result = provider.upsertRecord(rec, upsert_options);
        printf("upsert result: %y\n", result);
    }

    static del(AbstractDataProvider provider) {
        *hash<auto> where_cond = QdpCmd::getSearchExpression("delete");
        int rec_count = provider.deleteRecords(where_cond);
        printf("%d record%s deleted\n", rec_count, rec_count == 1 ? "" : "s");
    }

    static doRequest(AbstractDataProvider provider) {
        *hash<auto> req = QdpCmd::getHash("request");
        *hash<auto> options = QdpCmd::getHash("request-options");
        auto resp = provider.doRequest(req, options);
        if (!opts.verbose && resp.info."response-uri" && exists resp.body) {
            printf("%s: %N\n", resp.info."response-uri", resp.body);
        } else {
            printf("%N\n", resp);
        }
    }

    static doRequestSearch(AbstractDataProvider provider) {
        hash<auto> req = QdpCmd::getHash("request", True);
        hash<auto> where_cond = QdpCmd::getSearchExpression("request-search", True);
        *hash<auto> options = QdpCmd::getHash("request-options");
        AbstractDataProviderRecordIterator i = provider.requestSearchRecords(req, where_cond, options);
        map printf("%y\n", $1), i;
    }

    static request(AbstractDataProvider provider) {
        if (*hash<auto> req = QdpCmd::getHash("request")) {
            DataProviderDataContextHelper dch(req);
            QdpCmd::showType(provider, provider.getRequestTypeWithData(req));
        } else {
            QdpCmd::showType(provider, provider.getRequestType());
        }
        if (opts.verbose) {
            *hash<string, hash<DataProviderOptionInfo>> opts = provider.getRequestOptions();
            if (opts) {
                QdpCmd::showOptionHash(provider, opts);
            }
        }
    }

    static response(AbstractDataProvider provider) {
        if (*hash<auto> req = QdpCmd::getHash("request")) {
            DataProviderDataContextHelper dch(req);
            QdpCmd::showType(provider, provider.getResponseTypeWithData(req));
        } else {
            QdpCmd::showType(provider, provider.getResponseType());
        }
    }

    static showOptionHash(AbstractDataProvider prov, *hash<auto> req, string offset = "") {
        foreach hash<auto> i in (req.pairIterator()) {
            foreach AbstractDataProviderType type in (i.value.type) {
                if (i.value.type.lsize() > 1) {
                    printf("%s[%d]: %s %s\n", offset, $#, type.getName(), i.key);
                } else {
                    printf("%s%s %s\n", offset, type.getName(), i.key);
                }
                *hash<string, AbstractDataField> fields = type.getFields();
                if (fields) {
                    QdpCmd::showRecord(prov, fields, offset + "  ");
                }
            }
        }
    }

    static showType(*AbstractDataProvider prov) {
        # intentionally empty
    }

    static showType(*AbstractDataProvider prov, string msg, hash<DataProviderMessageInfo> info) {
        printf("%y: %s\n", msg, QdpCmd::getDesc(0, info));
        hash<string, bool> rmap;
        QdpCmd::showType(prov, \rmap, info.type);
    }

    static showType(*AbstractDataProvider prov, AbstractDataProviderType type, *string offset) {
        hash<string, bool> rmap;
        QdpCmd::showType(prov, \rmap, type, offset);
    }

    static showType(*AbstractDataProvider prov, reference<hash<string, bool>> rmap, AbstractDataProviderType type,
            *string offset) {
        *hash<string, AbstractDataField> fields = type.getFields();
        # get data context
        auto dctxt = DataProviderDataContextHelper::get();
        if (fields) {
            QdpCmd::showRecord(prov, \rmap, fields, offset, dctxt);
        } else {
            if (type.getBaseTypeCode() == NT_LIST && (*AbstractDataProviderType element_type = type.getElementType())
                && (*hash<string, AbstractDataField> element_rec = element_type.getFields())) {
                printf("%s%s elements ->\n", type.getName(), offset);
                QdpCmd::showRecord(prov, \rmap, element_rec, offset + "  ", dctxt[0]);
            } else {
                printf("%s%s%s (%s)\n", offset, type.getName());
            }
        }
    }

    static *hash<auto> getSearchExpression(string action, *bool required) {
        *hash<auto> rv = QdpCmd::getHash(action, required);
        if (opts.exp && rv.size() == 2 && rv.exp.typeCode() == NT_STRING && rv.hasKey("args")) {
            return QdpCmd::getExpression(rv.exp, rv.args);
        }
        return rv;
    }

    static hash<DataProviderExpression> getExpression(string exp, auto args) {
        return <DataProviderExpression>{
            "exp": exp,
            "args": QdpCmd::getExpressionArgs(args),
        };
    }

    static *softlist<auto> getExpressionArgs(auto args) {
        if (!exists args) {
            return;
        }
        list<auto> rv = ();
        foreach auto arg in (args) {
            if (arg.size() == 2 && arg.exp.typeCode() == NT_STRING && arg.hasKey("args")) {
                rv += QdpCmd::getExpression(arg.exp, arg.args);
            } else if (arg.field.typeCode() == NT_STRING) {
                rv += cast<hash<DataProviderFieldReference>>(arg);
            } else {
                push rv, arg;
            }
        }
        return rv;
    }

    static *hash<auto> getHash(string action, *bool required) {
        auto arg = shift ARGV;
        return getHashIntern(arg, action, required);
    }

    static *hash<auto> getHashIntern(auto arg, string action, *bool required) {
        if (exists arg) {
            arg = parse_to_qore_value(arg);
            if (exists arg && arg.typeCode() != NT_HASH) {
                QdpCmd::error("Invalid %s argument %y; expecting type \"hash\"; got type %y instead", action, arg,
                    arg.type());
            }
        }
        if (required && !exists arg) {
            QdpCmd::error("Missing required %s argument; expecting type \"hash\"", action);
        }
        return arg;
    }

    static *string getString(string action, *bool required) {
        auto arg = shift ARGV;
        if (exists arg) {
            return arg;
        }
        if (required && !arg) {
            QdpCmd::error("Missing required %s argument; expecting type \"string\"", action);
        }
    }

    static auto getAny(string action) {
        auto arg = shift ARGV;
        if (exists arg) {
            arg = parse_to_qore_value(arg);
        }
        return arg;
    }

    private showActions(string app) {
        if (!ENV.QORE_PROVIDER_INDEX_DIR) {
            DataProvider::registerKnownFactories();
        }
        *hash<string, hash<DataProviderActionInfo>> actions = DataProviderActionCatalog::getActionHash(app);
        if (!actions) {
            error("Application %y is unknown; known applications: %y", app, (map $1.name,
                DataProviderActionCatalog::getAllApps()));
        }
        printf("Application %y:\n", app);
        printf("- actions: %y\n", keys actions);
        if (opts.verbose) {
            hash<DataProviderAppInfo> info = DataProviderActionCatalog::getAppEx(app);
            if (info.scheme) {
                hash<ConnectionSchemeInfo> sinfo = ConnectionSchemeCache::getSchemeEx(info.scheme);
                printf("- schemes: %y\n", keys sinfo.schemes);
                *hash<auto> opts = map {$1.key: $1.value.default_value}, sinfo.options.pairIterator(),
                    exists $1.value.default_value;
                if (opts) {
                    printf("- connection options:\n");
                    map printf("  - %s: %y\n", $1.key, $1.value), opts.pairIterator();
                }
            }
        }
        exit(0);
    }

    private AbstractDataProvider getDataProviderForAppAction(string app_name, string action_name, *string opts) {
        if (!ENV.QORE_PROVIDER_INDEX_DIR) {
            DataProvider::registerKnownFactories();
        }
        *hash<string, hash<DataProviderActionInfo>> actions = DataProviderActionCatalog::getActionHash(app_name);
        if (!actions) {
            error("Application %y is unknown", app_name);
        }
        *hash<DataProviderActionInfo> action = actions{action_name};
        if (!action) {
            error("Application %y does not provide action %y; available actions: %y", app_name, action_name,
                keys actions);
        }
        *hash<auto> options;
        if (opts) {
            options = parse_to_qore_value(opts);
            if (exists options && options.typeCode() != NT_HASH) {
                throw "ACTION-ERROR", sprintf("option string for @%s/%s does not parse to type \"hash\"; got "
                    "type %y instead", app_name, action_name, options.type());
            }
        }
        hash<DataProviderAppInfo> app = DataProviderActionCatalog::getAppEx(app_name);

        string connection;
        AbstractConnection conn;
        if (app.scheme.val()) {
            connection = OptionHelper::getString(options, "connection", True);
            remove options.connection;
            conn = get_connection(connection);
            hash<ConnectionSchemeInfo> scheme_info = ConnectionSchemeCache::getSchemeEx(app.scheme);
            if (!scheme_info.schemes{string scheme = conn.getType()}) {
                throw "APP-CONFIG-ERROR", sprintf("app %y action %y: connection %y has scheme %y; expecting %y", app.name,
                    action.action, connection, scheme, keys scheme_info.schemes);
            }
            if (!conn.hasDataProvider()) {
                throw "APP-CONFIG-ERROR", sprintf("connection %y has no data provider but should be valid for "
                    "app %y action %y", connection, app.name, action.action);
            }
        }

        # check data provider path options
        foreach hash<auto> i in (action.path_vars.pairIterator()) {
            if (!options{i.key}.val()) {
                throw "ACTION-ARG-ERROR", sprintf("app %y action %y: required option %y is missing in \"options\"",
                    app.name, action.action, i.key);
            }
        }

        AbstractDataProvider prov;
        try {
            # NOTE: action.subtype may be NOTHING
            if (conn) {
                prov = conn.getDataProvider(action.subtype);
            } else if (action.cls) {
                prov = DataProviderActionCatalog::getDataProviderForAction(action, \options);
            }
            # set logger on data provider
            prov.setLogger(logger);

            DataProviderDataContextHelper dch(options);
            foreach string seg in (action.path[1..].split("/")) {
                if (*string var = (seg =~ x/^\{([^\}]+)\}$/)[0]) {
                    seg = remove options{var};
                }
                prov = prov.getChildProviderEx(seg);
            }
        } catch (hash<ExceptionInfo> ex) {
            rethrow "ACTION-ERROR", sprintf("error acquiring data provider for app %y action: %y: %s: %s", app.name,
                action.action, ex.err, ex.desc);
        }
        return prov;
    }

    private AbstractDataProvider getDataProvider(string name) {
        if (name =~ /{([^}]*)}/) {
            try {
                AbstractDataProvider dp = DataProvider::getFactoryObjectFromStringUseEnv(name);
                # set logger on data provider
                dp.setLogger(logger);
                return dp;
            } catch (hash<ExceptionInfo> ex) {
                if (ex.err == "FACTORY-ERROR") {
                    error("%s", ex.desc);
                }
                rethrow;
            }
        }

        # load providers from environment and try to load a connection
        try {
            *AbstractDataProvider dp = DataProvider::tryLoadProviderForConnectionFromEnv(name, opts.subtype,
                logger);
            if (dp) {
                # set logger on data provider
                dp.setLogger(logger);
                return dp;
            }
        } catch (hash<ExceptionInfo> ex) {
            if (ex.err == "DATA-PROVIDER-ERROR") {
                error("connection %y exists but does not support the data provider API", name);
            }
            rethrow;
        }

        error("no connection or datasource connection %y exists in any known data provider", name);
    }

    static createIndex() {
        *hash<auto> index = QdpCmd::getHash("index");
        try {
            hash<auto> info = ProviderIndex::createDataProviderIndex(
                index ? cast<hash<DataProviderIndexInfo>>(index) : NOTHING
            );
            printf("Created index %y: %d bytes in %y (%y)\n", info.path, info.size, info.delta, info.summary);
            exit(0);
        } catch (hash<ExceptionInfo> ex) {
            error("%s: %s", ex.err, ex.desc);
        }
    }

    static listApps() {
        DataProvider::registerKnownFactories();
        *list<hash<DataProviderAppInfo>> l = DataProviderActionCatalog::getAllApps();
        foreach hash<DataProviderAppInfo> app in (l) {
            # get actions for app
            *list<hash<DataProviderActionInfo>> actions = DataProviderActionCatalog::getActions(app.name);
            printf("%s: scheme %y, %d action%s: %s\n", app.name, app.scheme, actions.size(),
                actions.size() == 1 ? "" : "s", app.desc);
            if (opts.verbose) {
                map printf(" + %s (%s %y): %s\n", $1.action, ActionNameMap{$1.action_code}, $1.path, $1.short_desc),
                    actions;
            }
        }
        exit(0);
    }

    static listConnections() {
        # get connections
        *hash<string, string> h = map {$1.key: $1.value.url}, get_connection_hash(True).pairIterator();
        # add datasource connections
        h += get_ds_hash(True);
        if (!h) {
            printf("no connections are present\n");
        } else {
            if (opts.verbose) {
                map printf("%s: %s\n", $1.key, $1.value), h.pairIterator();
            } else {
                map printf("%s\n", $1), keys h;
            }
        }
        exit(0);
    }

    static listFactories() {
        # load known data provider factories
        DataProvider::registerKnownFactories();
        # load providers from environment
        DataProvider::loadProvidersFromEnvironment();
        *list<string> flist = DataProvider::listFactories();
        if (opts.verbose) {
            map printf("%s: %N\n", $1, DataProvider::getFactory($1).getInfoAsData()), flist;
        } else {
            printf("%y\n", flist);
        }
        exit(0);
    }

    static listTypes() {
        # load known data provider factories
        DataProvider::registerKnownTypes();
        # load types from environment
        DataProvider::loadTypesFromEnvironment();
        *list<string> tlist = DataProvider::listTypes();
        if (opts.verbose) {
            map printf("%s: %N\n", $1, DataProvider::getType($1).getInfo()), tlist;
        } else {
            printf("%y\n", tlist);
        }
        exit(0);
    }

    static usage() {
        printf("usage:
 %s [options] <connection>[/child1/child2...] <cmd> [args...]
 %s [options] <factory>{<options>}[/child1/child2...] <cmd> [args...]
 %s [options] @<app>/<action>[{<options>}] [args...]
  <cmd> = create|delete|dorequest|errors|info|list|record|request|response|reply|rsearch|search|update|upsert (default=list)
    action-info|ainfo|ai
        show info for the given app action (must use @<app>/<action> for the data provider)
    create <new record> (ex create id=123,name=\"my name\")
        create a new record
    example-error|exerror [error]
        show an example error response
    example-event|exevent [event]
        show an example event
    example-record|exrecord
        show an example record
    example-request|exrequest
        show an example request
    example-response|exresponse
        show an example response
    field-add|fadd <name> <field desc hash>
        adds a field to a data provider
    field-delete|fdelete <name>
        deletes a field from a data provider
    field-update|fupdate <old-name> <new-name> [<field desc hash>]
        deletes a field from a data provider
    delete <match criteria>
        deletes record matching the given criteria
    dorequest <request info>
        executes a request against the given provider (if supported)
    errors [<code>]
        lists all error replies
    event <event>
        shows the event type
    events
        list all supported event types
    info
        show information about the data provider
    interactive|ix
        listen to events generated by an observable data provider and send response messages
    list
        list child data provider names
    list-details|ldetails
        list child data providers with details
    list-events|levents
        list event types
    list-messages|lmessages|lmsgs
        list message types
    listen
        listen to events generated by an observable data provider
    message|msg <message>
        shows the message type
    provider-create|pcreate <name> <desc hash> [<option hash>]
        create a new data provider
    provider-delete|pdelete <name>
        delete a child data provider
    record
        show the record format (more -v's = more info)
    request
        show request information (if supported)
    response|reply
        show successful response information (if supported)
    rsearch <req> <search> [<options>]
        executes a request and then a search on the results (if supported)
    search [search criteria] (ex: search name=\"my name\")
        search for record(s) matching the given criteria
    send-message|send-msg|smsg <message id> [<message payload>] [<send options>]
        sends a message
    show-providers|sp
        shows data provider endpoints from the given root
    update <set criteria> <match criteria> (ex update name=other id=123)
        update the given records with the given information
    upsert <record> (ex update id=123,name=\"my name\")
        upserts the given record

 -a,--list-apps            list applications
 -b,--bulk                 use the bulk API for searches
 -c,--list-connections     list known connections and exit
 -f,--list-factories       list known data provider factories and exit
 -H,--host=ARG             set the hostname (+ optionally port) for webhooks
 -I,--create-index         create data provider indices in QORE_PROVIDER_INDEX_DIR
 -m,--module=ARG           Data Provider modules to load manually
 -o,--options=ARG          set implied options for a data provider (must parse to a hash in Qore format)
 -R,--show-required        show only required attributes when showing types
 -S,--servers              enable data provider background threads and processes
 -s,--subtype=ARG          use Data Provider subtype when creating from a connection
 -t,--list-types           list known data provider types and exit
 -v,--verbose[=ARG]        show more output
 -x,--expressions          create search expressions when searching
 -h,--help                 this help text
", get_script_name(), get_script_name(), get_script_name());
        exit(1);
    }

    static error(string fmt) {
        stderr.printf("%s: ERROR: %s\n", get_script_name(), vsprintf(fmt, argv));
        exit(1);
    }
}

class Term {
    public {}
    private {
        TermIOS orig;
    }

    constructor() {
        TermIOS t();

        # get current terminal attributes for stdin
        stdin.getTerminalAttributes(t);

        # save a copy
        orig = t.copy();

        # get local flags
        int lflag = t.getLFlag();

        # disable canonical input mode (= turn on "raw" mode)
        lflag &= ~ICANON;

        # turn off echo mode
        lflag &= ~ECHO;

        # do not check for special input characters (INTR, QUIT, and SUSP)
        lflag &= ~ISIG;

        # set the new local flags
        t.setLFlag(lflag);

        # set minimum characters to return on a read
        t.setCC(VMIN, 1);

        # set character input timer in 0.1 second increments (= no timer)
        t.setCC(VTIME, 0);

        # make these terminal attributes active
        stdin.setTerminalAttributes(TCSADRAIN, t);
    }

    destructor() {
        restore();
    }

    restore() {
        # restore terminal attributes
        stdin.setTerminalAttributes(TCSADRAIN, orig);
    }
}

class MyObserver inherits Observer {
    public {}

    private {
        hash<DataProviderInfo> info;
        TermIOS t();
    }

    constructor(hash<DataProviderInfo> info) {
        self.info = info;
        # get current terminal attributes for stdin
        stdin.getTerminalAttributes(t);
    }

    update(string event_id, hash<auto> data_) {
        string msg = sprintf("received event %y:", event_id);
        if (!data_) {
            msg += " no data";
        }
        if (data_) {
            msg += sprintf("\n  %N", data_);
        }
        log("<", msg);
        if (info.connection_event == event_id) {
            log("<", "connected to server");
        } else if (info.disconnection_event == event_id) {
            if (info.supports_auto_reconnect) {
                log("<", "disconnected from server%s", QdpCmd::ix ? "; trying to reconnect..." : "");
            } else {
                log("<", "disconnected from server; auto_reconnect not enabled; exiting");
                # restore original terminal attributes
                stdin.setTerminalAttributes(TCSADRAIN, t);
                exit(0);
            }
        }
        if (QdpCmd::ix) {
            stdout.print("> ");
        }
        stdout.sync();
    }

    log(string inout, string fmt, ...) {
        if (QdpCmd::ix) {
            stdout.print("\r");
        }
        fmt = sprintf("%s %s ", inout, now_us().format("YYYY-MM-DD HH:mm:SS.xx")) + fmt;
        string msg = vsprintf(fmt, argv);
%ifndef NoLinenoise
        msg =~ s/\n/\015\012/g;
%endif
        stdout.print(msg + "\n");
        if (QdpCmd::ix) {
            stdout.print("\r");
        }
    }
}

class ConsoleAppender inherits LoggerAppenderWithLayout {
    constructor() : LoggerAppenderWithLayout("console", new LoggerLayoutPattern("%d{YYYY-MM-DD HH:mm:SS.xx} T%t [%p]: %m%n")) {
        open();
    }

    processEventImpl(int type, auto params) {
        switch (type) {
            case EVENT_LOG:
%ifndef NoLinenoise
                params =~ s/\n/\r\n/g;
%endif
                print(params);
                break;
        }
    }
}

class InputHelper {
    public {
        AbstractDataProvider provider;
        # message map
        hash<string, hash<DataProviderMessageInfo>> mmap;

%ifndef NoLinenoise
        static string path;
        const HistoryFile = ".qdp-history";
%endif

        const Prompt = "> ";

        const Commands = {
            "exit": True,
            "help": True,
            "history": True,
            "info": True,
            "quit": True,
        };
    }

    constructor(AbstractDataProvider provider) {
        self.provider = provider;
        mmap = provider.getMessageTypes();

%ifndef NoLinenoise
        Linenoise::history_set_max_len(100);
        Linenoise::set_callback(\completion());

        path = ENV.HOME + DirSep + HistoryFile;

        bool ok;
        try {
            # create history file if it doesn't exist
            if (!is_file(path)) {
                File f();
                f.open2(path, O_CREAT | O_WRONLY);
            }
            ok = True;
        } catch (hash<ExceptionInfo> ex) {
            stderr.printf("%y: WARNING: cannot create history file: %s: %s\n", path, ex.err, ex.desc);
        }

        if (ok) {
            try {
                Linenoise::history_load(path);
            } catch (hash<ExceptionInfo> ex) {
                stderr.printf("%y: WARNING: cannot load history: %s: %s\n", path, ex.err, ex.desc);
            }
        }
%endif
    }

    destructor() {
%ifndef NoLinenoise
        Linenoise::history_save(path);
%endif
    }

    string get() {
        string line;
        while (True) {
%ifndef NoLinenoise
            *string line0 = Linenoise::line(Prompt);
            if (!exists line0) {
                line = "exit";
                break;
            }
            line = line0;
            Linenoise::history_add(line);
%else
            stdout.print(Prompt);
            stdout.sync();
            line = trim(stdin.readLine());
%endif

%ifndef NoLinenoise
                # save history
                Linenoise::history_save(InputHelper::path);
%endif

            if (line == 'help' || line == '?') {
                printf("commands:\n  help\n  info\n  quit\n  history\n");
                continue;
            }
            if (line == "info") {
                string msg = sprintf("using: %y: %s", provider.getName(), provider.getShortDesc());
                *hash<DataProviderConnectionInfo> cinfo = provider.getConnectionInfo();
                if (cinfo.url) {
                    msg += sprintf(" (%s connected: %y auto reconnect: %y)", cinfo.url, cinfo.connected,
                        cinfo.auto_reconnect);
                }
                print(msg + "\n");
                if (cinfo.info) {
                    printf("info: %N\n", cinfo.info);
                }
                print("messages:\n");
                map (printf("+ %y: ", $1.key), QdpCmd::showType(provider, $1.value.type)),
                    provider.getMessageTypes().pairIterator();
                print("events:\n");
                map (printf("+ %y: ", $1.key), QdpCmd::showType(provider, $1.value.type)),
                    provider.getEventTypes().pairIterator();
                continue;
            }
%ifndef NoLinenoise
            if (line == 'history') {
                map printf("%s\n", $1), Linenoise::history();
                continue;
            }
%endif

            if (!line.val()) {
                continue;
            }

            break;
        }

        return line;
    }

    softlist<string> completion(string str) {
        list<string> rv = ();
        rv += map $1, keys Commands, $1.equalPartial(str);
        rv += map $1 + ": ", keys mmap, $1.equalPartial(str);
        return rv;
    }
}

class WebhookHandler inherits RestHandler {
    private {
        string rest_method;
        string root_path;
        code callback;

        static bool cors_enable = DefaultCorsEnable;
        static *softlist<string> cors_accept_origin;
        static *string cors_allow_methods = foldl $1 + ", " + $2, DefaultCorsAllowMethods;
        static softstring cors_max_age = DefaultCorsMaxAge;
        static *string cors_allow_headers = foldl $1 + ", " + $2, DefaultCorsAllowHeaders;
        static bool cors_allow_credentials = DefaultCorsAllowCredentials;
        static bool cors_accept_all_origins = DefaultCorsAcceptAllOrigins;
        static hash<string, bool> cors_origin_map;

        const DefaultCorsEnable = True;
        const DefaultCorsAllowMethods = ("GET", "POST", "PUT", "PATCH", "DELETE", "OPTIONS");
        const DefaultCorsMaxAge = 9999999;
        const DefaultCorsAllowHeaders = (
            "Content-Type", "content-type", "Content-Language", "content-language",
            "Accept", "Accept-Language", "Authorization", "Qorus-Token", "Qorus-Client-Time-Zone",
        );
        const DefaultCorsAllowCredentials = True;
        const DefaultCorsAcceptAllOrigins = True;
    }

    constructor(string method, string path, code callback) : RestHandler() {
        rest_method = method;
        root_path = (path =~ /^\//) ? path[1..] : path;
        self.callback = callback;
    }

    removeRootPath(reference<string> path) {
        if (path.startsWith(root_path)) {
            splice path, 0, root_path.size();
        }
    }

    hash<HttpHandlerResponseInfo> get(hash<auto> cx, *hash<auto> ah) {
        return callWebhook(cx, ah);
    }

    hash<HttpHandlerResponseInfo> post(hash<auto> cx, *hash<auto> ah) {
        return callWebhook(cx, ah);
    }

    hash<HttpHandlerResponseInfo> put(hash<auto> cx, *hash<auto> ah) {
        return callWebhook(cx, ah);
    }

    hash<HttpResponseInfo> handleRequest(HttpListenerInterface listener, Socket s, hash<auto> cx, hash<auto> hdr,
            *data b) {
        #logDebug("Webhook headers: %N", hdr);

        # handle OPTIONS requests
        if (hdr.method == "OPTIONS" && (*hash<auto> cors_hdrs = getCorsResponseHeaders(cx))) {
            # return a CORS response
            return AbstractHttpRequestHandler::makeResponse(200, NOTHING, cors_hdrs);
        }

        # check for valid method
        if (hdr.method != rest_method) {
            # get rest class name
            string path = cx.url.path ?? "";
            removeRootPath(\path);
            return AbstractHttpRequestHandler::makeResponse(405, sprintf("HTTP method %y is not supported with URI "
                "path %y; supported method: %s", hdr.method, path, rest_method), getCorsResponseHeaders(cx) +
                {"Allow": rest_method});
        }

        cx.sctx.listener = listener;
        hash<HttpResponseInfo> rv = RestHandler::handleRequest(listener, s, cx, hdr, b);
        if (!rv.hdr."Access-Control-Allow-Origin") {
            rv.hdr += getCorsResponseHeaders(cx);
        }
        return rv;
    }

    hash<HttpHandlerResponseInfo> callWebhook(hash<auto> cx, *hash<auto> ah) {
        logInfo("Webhook %y called", root_path);
        logDebug("Webhook %y args: %y", root_path, ah);
        on_error QdpCmd::http.logError("%s", get_exception_string($1));
        try {
            return <HttpHandlerResponseInfo>{
                "code": 200,
                "body": callback(ah),
            };
        } catch (hash<ExceptionInfo> ex) {
            QdpCmd::http.logError("Webhook %y args: %y error: %s", root_path, ah, get_exception_string(ex));
            return <HttpHandlerResponseInfo>{
                "code": 500,
                "body": sprintf("%s: %s", ex.err, ex.desc),
            };
        }
    }

    private hash<HttpHandlerResponseInfo> returnRestException(hash<ExceptionInfo> ex) {
        try {
            if (ex.arg.typeCode() == NT_OBJECT) {
                remove ex.arg;
            }
        } catch (hash<ExceptionInfo> ex0) {
            remove ex.arg;
        }
        if (ex.err =~ /-ARG-ERROR$/) {
            return <HttpHandlerResponseInfo>{
                "code": 400,
                "body": ex.desc,
            };
        }
        if (ex.err == "AUTHORIZATION-ERROR" || ex.err =~ /-ACCESS-ERROR$/) {
            return <HttpHandlerResponseInfo>{
                "code": 403,
                "body": ex,
            };
        }
        if (ex.err == "REST-NOT-FOUND-ERROR") {
            return <HttpHandlerResponseInfo>{
                "code": 404,
                "body": ex.desc,
            };
        }
        # for all other errors, return the standard "409 Conflict" response
        return RestHandler::returnRestException(ex);
    }

    private *hash<auto> errorResponseHeaders(hash<auto> cx) {
        return getCorsResponseHeaders(cx);
    }

    static *hash<auto> getCorsResponseHeaders(hash<auto> cx) {
        bool explicit_match;
        if (!WebhookHandler::corsOriginOk(cx, \explicit_match)) {
            return;
        }
        return WebhookHandler::getCorsResponseHeadersIntern(cx, explicit_match);
    }

    static private hash<auto> getCorsResponseHeadersIntern(hash<auto> cx, *bool explicit_match) {
        hash<auto> rv;
        # https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Access-Control-Allow-Origin
        # if the server supports clients from multiple origins, it must return the origin for the specific client
        # making the request
        if (cx.hdr.origin && !cors_accept_all_origins) {
            rv = {
                "Access-Control-Allow-Origin": cx.hdr.origin,
                "Vary": "Origin",
            };
        } else {
            rv = {
                "Access-Control-Allow-Origin": "*",
            };
        }
        # return other headers for preflight (OPTIONS) requests
        if (cx.hdr.method == "OPTIONS") {
            rv += {
                "Access-Control-Allow-Methods": cors_allow_methods,
                "Access-Control-Max-Age": cors_max_age,
            };
            if (cors_allow_headers) {
                rv."Access-Control-Allow-Headers" = cors_allow_headers;
            }
        }
        if (cors_allow_credentials) {
            rv."Access-Control-Allow-Credentials" = "true";
        }
        return rv;
    }

    static bool corsOriginOk(hash<auto> cx, *reference<bool> explicit_match) {
        if (!cors_enable) {
            return False;
        }
        if (cors_accept_all_origins) {
            return True;
        }
        if (cors_origin_map{cx.hdr.origin}) {
            explicit_match = True;
            return True;
        }
        return False;
    }

    static corsUpdateOption(string opt, auto value) {
        switch (opt) {
            case "cors-enable":
                cors_enable = value;
                break;

            case "cors-allow-origin": {
                cors_accept_origin = value;
                cors_accept_all_origins = False;
                foreach string origin in (cors_accept_origin) {
                    if (origin == "*") {
                        cors_accept_all_origins = True;
                    }
                    cors_origin_map{origin} = True;
                }
                break;
            }

            case "cors-allow-methods":
                cors_allow_methods = foldl $1 + ", " + $2, value;
                break;

            case "cors-max-age":
                cors_max_age = value.toString();
                break;

            case "cors-allow-headers":
                cors_allow_headers = foldl $1 + ", " + $2, value;
                break;

            case "cors-allow-credentials":
                cors_allow_credentials = value;
                break;
        }
    }
}