Bootstrap

+name: Run Tests and Analysis
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    branches:
+      - main
+
+jobs:
+  build-linux:
+    strategy:
+      matrix:
+        python-version: ["3.10"]
+    runs-on: "ubuntu-latest"
+    steps:
+      - name: Install graphviz
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y graphviz
+
+      - uses: actions/checkout@v3
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Remove development dependencies
+        run: sed -i '/\[tool.poetry.group.dev.dependencies\]/,/\[/d' pyproject.toml
+
+      - name: Install Poetry
+        uses: snok/install-poetry@v1
+        with:
+          version: 1.2.2
+          virtualenvs-in-project: true
+
+      - name: Install dependencies
+        run: poetry install --no-interaction --no-root
+
+      - name: Install library
+        run: poetry install --no-interaction
+
+      - name: Install development tools
+        run: poetry add gmpg
+
+      - uses: psf/black@stable
+        with:
+          options: "--check --verbose"
+          src: "."
+          version: "23.7"
+
+      - uses: isort/isort-action@v1
+        with:
+          configuration: "--profile black"
+
+      - run: echo "$(poetry env info --path)/bin" >> $GITHUB_PATH
+      - uses: jakebailey/pyright-action@v1
+
+      # - name: Run tests
+      #   run: poetry run gmpg test
+
+      - name: Run analysis
+        run: poetry run gmpg analyze
+
+      - name: Generate dependency graph
+        run: poetry run gmpg graph
+
+      - uses: actions/upload-artifact@v3
+        with:
+          name: analysis
+          path: |
+            test_coverage.xml
+            test_results.xml
+            api_python.json
+            deps.svg

+[tool.poetry]
+name = "webint-sites"
+version = "0.0.2"
+description = "manage sites from your website"
+authors = ["Angelo Gladding <angelo@ragt.ag>"]
+license = "AGPL-3.0-or-later"
+packages = [{include="webint_sites"}]
+
+[tool.poetry.plugins."webapps"]
+sites = "webint_sites:app"
+
+[tool.poetry.dependencies]
+python = ">=3.10,<3.11"
+webint = ">=0.0"
+
+[tool.poetry.group.dev.dependencies]
+gmpg = {path="../gmpg", develop=true}
+bgq = {path="../bgq", develop=true}
+newmath = {path="../newmath", develop=true}
+sqlyte = {path="../sqlyte", develop=true}
+webagt = {path="../webagt", develop=true}
+webint = {path="../webint", develop=true}
+micropub = {path="../python-micropub", develop=true}
+
+# [[tool.poetry.source]]
+# name = "main"
+# url = "https://ragt.ag/code/pypi"
+
+[tool.pyright]
+reportGeneralTypeIssues = false
+reportOptionalMemberAccess = false
+
+[build-system]
+requires = ["poetry-core>=1.0.0"]
+build-backend = "poetry.core.masonry.api"

+"""Manage sites from your website."""
+
+import web
+from web import host, tx
+
+app = web.application(
+    __name__,
+    prefix="sites",
+    args={"machine": r"\w+", "domain_name": r"\w+"},
+    model={
+        "machines": {
+            "name": "TEXT UNIQUE",
+            "ip_address": "TEXT UNIQUE",
+            "details": "JSON",
+        },
+        "domains": {
+            "name": "TEXT UNIQUE",
+            "nameserver": "TEXT UNIQUE",
+            "details": "JSON",
+        },
+    },
+)
+
+
+def spawn_machine(name, config):
+    """Spin up a VPS and setup a machine."""
+    machine, secret = web.host.setup_website(
+        name, "canopy-platform", "canopy:app", config
+    )
+    print(machine, secret)
+    tx.db.insert("machines", name=name, ip_address=machine.address, details={})
+
+
+def build(ip_address, program):
+    details = tx.db.select("machines", where="ip_address = ?", vals=[ip_address])[0][
+        "details"
+    ]
+    details[program] = getattr(host, f"setup_{program}")(ip_address)
+    tx.db.update("machines", where="ip_address = ?", vals=[ip_address], details=details)
+
+
+@app.control("")
+class Sites:
+    """Manage your websites."""
+
+    owner_only = ["get"]
+
+    def get(self):
+        machines = tx.db.select("machines")
+        return app.view.index(machines)
+
+
+@app.control("gaea")
+class Gaea:
+    """"""
+
+    def get(self):
+        # XXX config = host.get_config()
+        # XXX if web.tx.request.headers["accept"] == "application/json":
+        # XXX     return config
+
+        # domains = []
+        # if "registrar" in config:
+        #     registrar = dict(config["registrar"])
+        #     clients = {
+        #         "dynadot.com": host.providers.Dynadot,
+        #         "name.com": host.providers.NameCom,
+        #     }
+        #     domains = clients[registrar.pop("provider")](**registrar).list_domains()
+        # dns = {}
+        # if "domain" in config:
+        #     dns["ns"] = [
+        #         str(ns)
+        #         for ns in web.dns.resolve(config["domain"], "NS").rrset.items.keys()
+        #     ]
+        #     dns["a"] = web.dns.resolve(config["domain"], "A")[0].address
+        return app.view.gaea()  # config)  # , domains, dns)
+
+    def post(self):
+        # form = web.form("subdomain", "name")
+        form = web.form("provider", "token")
+        config = host.get_config()
+
+        # domain = config["domain"]
+        # fqdn = domain
+        # if form.subdomain:
+        #     fqdn = f"{form.subdomain}.{domain}"
+        # update_config(fqdn=fqdn)
+
+        if form.provider == "digitalocean.com":
+            c = host.digitalocean.Client(form.token)
+            try:
+                c.get_keys()
+            except host.digitalocean.TokenError:
+                return {"status": "error", "message": "bad token"}
+        elif form.provider == "linode.com":
+            ...
+        elif form.provider == "hetzner.com":
+            ...
+        else:
+            return {
+                "status": "error",
+                "message": f"unsupported provider: {form.provider}",
+            }
+        config = host.update_config(
+            host={
+                "provider": form.provider,
+                "token": form.token,
+            }
+        )
+
+        # try:
+        #     ip_address = config["ip_address"]
+        # except KeyError:
+        ip_address = host.spawn_machine("canopy", config["host"]["token"])
+        config = host.update_config(ipAddress=ip_address, status={})
+
+        # registrar = config["registrar"]
+        # if registrar["provider"] == "dynadot.com":
+        #     dynadot = host.providers.Dynadot(registrar["token"])
+        #     dynadot.create_record(domain, ip_address, form.subdomain)
+        # elif registrar["provider"] == "name.com":
+        #     namecom = host.providers.NameCom(registrar["username"],
+        #                                      registrar["token"])
+        #     namecom.create_record(domain, ip_address, form.subdomain)
+
+        host.setup_machine(ip_address)
+        config = host.update_config(system_setup=True)
+        host.setup_nginx(ip_address)
+        config = host.update_config(nginx_installed=True)
+        return
+        host.generate_dhparam(ip_address)
+        host.setup_tor(ip_address)
+        host.setup_python(ip_address)
+        host.setup_supervisor(ip_address)
+
+        # while True:
+        #     try:
+        #         if web.dns.resolve(fqdn, "A")[0].address == ip_address:
+        #             break
+        #     except (
+        #         web.dns.NoAnswer,
+        #         web.dns.NXDOMAIN,
+        #         web.dns.Timeout,
+        #         web.dns.NoNameservers,
+        #     ):
+        #         pass
+        #     else:
+        #         print("waiting for DNS changes to propagate..")
+        #         time.sleep(15)
+
+        onion, passphrase = setup_canopy(ip_address)  # , form.name, fqdn)
+        update_config(onion=onion, passphrase=passphrase)
+        return {"onion": onion, "passphrase": passphrase}
+
+
+@app.control("machines")
+class Machines:
+    """Manage your machines."""
+
+    def get(self):
+        return app.view.machines()
+
+    def post(self):
+        name = web.form("name").name
+        config = {
+            "host": "digitalocean",
+            "host_token": tx.app.cfg["DIGITALOCEAN_TOKEN"],
+        }
+        # token = tx.db.select(
+        #     "providers", where="service = ?", vals=["digitalocean.com"]
+        # )[0]["token"]
+        web.enqueue(spawn_machine, name, config)
+        raise web.Accepted("machine is being created..")
+
+
+@app.control("machines/{machine}")
+class Machine:
+    """Manage one of your machines."""
+
+    def get(self, machine):
+        details = tx.db.select("machines", where="name = ?", vals=[machine])[0]
+        # try:
+        #     token = get_dynadot_token(tx.db)
+        # except IndexError:
+        #     domains = None
+        # else:
+        #     domains = host.dynadot.Client(token).list_domain()
+        root_ssh = web.host.Machine(details["ip_address"], "root", "gaea_key").run
+        gaea_ssh = web.host.Machine(details["ip_address"], "root", "gaea_key").run
+        return app.view.machine(details, root_ssh, gaea_ssh)
+
+
+@app.control("machines/{machine}/update")
+class MachineBuild:
+    """Manage your machine's system updates."""
+
+    def get(self):
+        ...
+
+    def post(self):
+        machine = tx.db.select("machines", where="name = ?", vals=[self.machine])[0]
+        web.enqueue(host.upgrade_system, machine["ip_address"])
+        raise web.Accepted("system is updating on the machine..")
+
+
+@app.control("machines/{machine}/builds")
+class MachineBuild:
+    """Manage your machine's builds."""
+
+    def get(self):
+        ...
+
+    def post(self):
+        program = web.form("program").program
+        machine = tx.db.select("machines", where="name = ?", vals=[self.machine])[0]
+        web.enqueue(build, machine["ip_address"], program)
+        raise web.Accepted(f"{program} is building on the machine..")
+
+
+@app.control("domains")
+class Domains:
+    """Manage your domains."""
+
+    def get(self):
+        return app.view.domains()
+
+    def post(self):
+        name = web.form("name").name
+        token = tx.db.select("providers", where="service = ?", vals=["dynadot.com"])[0][
+            "token"
+        ]
+        # XXX web.enqueue(spawn_machine, name, token)
+        raise web.Created("domain has been added..", f"/sites/domains/{name}")
+
+
+@app.control("domains/{domain_name}")
+class Domain:
+    """Manage one of your domains."""
+
+    def get(self):
+        domain = tx.db.select("domains", where="name = ?", vals=[self.domain_name])[0]
+        return app.view.domain(domain)

+from web import tx
+
+__all__ = ["tx"]

+$def with ()
+$# , domains, dns)
+<!doctype html>
+<html>
+<head>
+<meta charset=utf-8>
+<title>Spawn your personal website..</title>
+<link rel=stylesheet href=/static/screen.css>
+</head>
+<body>
+<h1>Spawn your personal website..</h1>
+<p>This installer will walk you through setting up your personal website.</p>
+
+$# $if "machine" in config:
+$#     <p>Machine: $config["machine"] <button>Delete</button></p>
+$# $else:
+    <form id=spawn>
+    <!--h3><input type=radio name=mode value=automatic checked disabled> Automatic</h3-->
+    <h2>Choose a host</h2>
+    <h3>In the cloud</h3>
+    <p>A cloud host is an organization that will manage an internet-connected
+    machine on your behalf typically for a monthly fee.</p>
+
+    <div id=hosts style="display:grid;grid-template-columns:50% 50%;">
+    <div>
+    <ul>
+    <li><label><input required type=radio name=provider value=digitalocean.com> DigitalOcean
+    <small><a href=https://digitalocean.com
+    target=_blank>digitalocean.com</a></small></label></li>
+    <li><label><input required type=radio name=provider value=linode.com> Linode
+    <small><a href=https://linode.com
+    target=_blank>linode.com</a></small></label></li>
+    <li><label><input required type=radio name=provider value=hetzner.com> Hetzner
+    <small><a href=https://hetzner.com
+    target=_blank>hetzner.com</a></small></label></legend></li>
+    </ul>
+    <label>API Token<br>
+    <label class=bounding><input required type=text name=token></label></label>
+    </div>
+
+    <div><!-- style=break-inside:avoid-->
+    <p id=digitalocean>Go to the <a
+    href=https://cloud.digitalocean.com/account/api/tokens target=_blank>Applications
+    &amp; API</a> page and generate a new personal access token with write scope.</p>
+    <p id=linode>Go to the <a
+    href=https://cloud.linode.com/profile/tokens target=_blank>Tokens</a>
+    page and create a new personal access token with full read/write access.</p>
+    <p id=hetzner>Go to the <a
+    href=https://console.hetzner.cloud/projects target=_blank>Projects</a> page,
+    create a new project, go to that project's page, click on "Security" &rarr;
+    "API Tokens" and generate a new token with read/write permission.</p>
+    </div>
+    </div>
+
+    <!--h3><input type=radio name=mode value=manual disabled> Manual</h3>
+    <p>The machine should be a fresh install of vanilla Debian or Raspberry Pi OS
+    (formerly Raspbian).</p>
+
+    <label>IP Address
+    <small>of Raspberry Pi / Virtual Machine / localhost / remote</small><br>
+    <label class=bounding><input type=text name=host_ip disabled></label></label><br>
+    <label>Root Passphrase<br>
+    <label class=bounding><input type=text name=root_passphrase disabled></label>
+    </label-->
+
+    <div class=buttons><button>Spawn</button></div>
+    </form>
+    <div id=progress>
+    <p id=machine_spawned></p>
+    <p id=system_setup></p>
+    <p id=nginx_setup></p>
+    </div>
+    <p id=status></p>
+
+$# $if "registrar" in config:
+$#     <p>Registrar: $config["registrar"]["provider"]</p>
+$# $else:
+$#     <form id=registrar>
+$#         <fieldset><legend><label><input type=checkbox name=use_domain checked> Domain Name <small>(optional)</small></label></legend>
+$#     <p>A domain name registrar is an organization that will register domain names
+$#     for you typically for an annual fee.</p>
+$#     <p><small>Note: in order to use a domain name your machine must be
+$#     publicly reachable.</small></p>
+$#     <ul>
+$#     <li><label><input type=radio name=provider value=dynadot.com> Dynadot
+$#     <small><a href=https://dynadot.com>dynadot.com</a></small></label></li>
+$#     <li><label><input type=radio name=provider value=name.com> Name
+$#     <small><a href=https://name.com>name.com</a></small></label></li>
+$#     </ul>
+$#     <p id=dynadot>Go to the <a
+$#     href=https://www.dynadot.com/account/domain/setting/api.html target=_blank>API</a>
+$#     page, unlock your account and generate a new key.</p>
+$#     <p id=namecom>Go to the <a
+$#     href=https://www.name.com/account/settings/api target=_blank>API</a>
+$#     page, agree to the terms if you haven't already and generate a new token.</p>
+$#     <label id=username class=field>Username: <label class=bounding><input
+$#     type=text name=username></label></label>
+$#     <label id=token class=field>Token/Key: <label class=bounding><input
+$#     type=text name=token></label></label>
+$#     <div class=buttons><button>Save</button></div>
+$#     </fieldset>
+
+$# $if "domain" in config:
+$#     $if not dns["ns"][0].endswith(config["registrar"]["provider"] + "."):
+$#         <p>Your DNS is being managed off-registrar... not supported yet.</p>
+$#     <form id=create>
+$#     <fieldset><legend>Site Details</legend>
+$#     <p>Domain<br>
+$#     <input type=text name=domain value=$config["domain"] disabled></p>
+$#     <p><label>Subdomain <small>(optional)</small><br><label
+$#     class=bounding><input name=subdomain></label></label></p>
+$#     <label class=field>Your display name<br><label class=bounding><input type=text
+$#     name=name></label></label>
+$#     <div class=buttons><button>Create</button></div>
+$#     </fieldset>
+$#     </form>
+$# $else:
+$#     <form id=domain>
+$#     <select name=domain>
+$#     $for domain, expiration in domains:
+$#         <option value=$domain>$domain</option>
+$#     </select>
+$#     <div class=buttons><button>Choose</button></div>
+$#     </form>
+
+<script src=/static/main.js></script>
+</body>
+</html>

+$def with (machines)
+$var title: Sites
+
+<p>list domains/apps and the machine they're on</p>
+
+$def aside():
+    $if "DIGITALOCEAN_TOKEN" in tx.app.cfg:
+        <h4>Machines</h4>
+        <ul id=machines>
+        $for machine in machines:
+            <li><a href=/sites/machines/$machine["name"]>$machine["name"]</a><br>
+            <small>$machine["ip_address"]</small></li>
+        </ul>
+        <form method=post action=/sites/machines>
+        <label><small>Name</small><br>
+        <label class=bounding><input type=text name=name></label></label>
+        <div class=buttons><button>Spawn Machine</button></div>
+        </form>
+    $else:
+        <p>Add <code style="background-color:#073642;border-radius:.25em;font-weight:bold;padding:0 .25em .125em .25em;"><small><small>DIGITALOCEAN_TOKEN</small></small></code> to <a
+        href=/system#settings>system settings</a></p>
+$var aside = aside
+
+<style>
+ul#machines {
+  list-style: none;
+  padding-left: 0; }
+</style>

+$def with (machine, root_ssh, gaea_ssh)
+$var breadcrumbs = ("machines", "Machines")
+$var title: $machine["name"]
+
+<p>$machine["ip_address"]</p>
+
+$ d = machine["details"]
+$# <p>$d</p>
+
+$code:
+    def get_output(cmd):
+        output = root_ssh(cmd).stdout
+        if "\n" not in output:
+            output += "\n"
+        return output.rpartition("\n")[0]
+
+    def get_gaea_output(cmd):
+        output = gaea_ssh(cmd).stdout
+        if "\n" not in output:
+            output += "\n"
+        return output.rpartition("\n")[0]
+
+$def render(cmd):
+    <pre style=font-size:.8em>$get_output(cmd)</pre>
+
+$def gaea_render(cmd):
+    <pre style=font-size:.8em>$get_gaea_output(cmd)</pre>
+
+<h3>Application</h3>
+$:render("/home/gaea/runinenv /home/gaea/app python -V")
+$:render("du -sh /home/gaea/app")
+<pre style=font-size:.8em>\
+$for line in get_output("ls -lthr /home/gaea/app/run | cut -d' ' -f5-").splitlines():
+    $if not line: $continue
+    $ size, month, day, yeartime, path = line.split()
+    $month $day $yeartime \
+    $if size:
+        <a href=/machines/dev/$path>\
+    $path\
+    $if size:
+        </a>\
+     <small>$size</small>
+</pre>
+
+<h4>Supervisor</h4>
+$:render("supervisorctl status")
+
+$def render_status(cmd):
+    $ status = {}
+    $for line in root_ssh(cmd).stdout.rpartition("\n")[0].splitlines():
+        $ key, _, value = line.partition("=")
+        $ status[key] = value
+    <pre style=font-size:.8em>$status["ActiveState"]</pre>
+
+<h3>Nginx</h3>
+$:render_status("systemctl --no-pager show nginx")
+
+<h3>Tmux</h3>
+$:gaea_render("tmux list-sessions")
+$:gaea_render("tmux list-windows -t 0")
+
+<h3>System</h3>
+$:render("df -ht ext4")
+$:render("grep 'Invalid user' /var/log/auth.log | cut -d' ' -f8 | sort | uniq -c | sort -r | head")
+
+$# <p>Last updated: $d.get("updated")</p>
+$# <form method=post action=/sites/machines/$machine["name"]/update>
+$# <div class=buttons><button>Update System</button></div>
+$# </form>
+$# 
+$# $if d.get("supervisor"):
+$#     <p>Supervisor: $d["supervisor"]</p>
+$# $else:
+$#     <form method=post action=/sites/machines/$machine["name"]/builds>
+$#     <input type=hidden name=program value=supervisor>
+$#     <div class=buttons><button>Install Supervisor</button></div>
+$#     </form>
+$# 
+$# $if d.get("nginx"):
+$#     <p>Nginx: $d["nginx"]</p>
+$# $else:
+$#     <form method=post action=/sites/machines/$machine["name"]/builds>
+$#     <input type=hidden name=program value=nginx>
+$#     <div class=buttons><button>Install Nginx</button></div>
+$#     </form>
+$# 
+$# $if d.get("tor"):
+$#     <p>Tor: $d["tor"]</p>
+$# $else:
+$#     <form method=post action=/sites/machines/$machine["name"]/builds>
+$#     <input type=hidden name=program value=tor>
+$#     <div class=buttons><button>Install Tor</button></div>
+$#     </form>
+$# 
+$# $if d.get("python"):
+$#     <p>Python: $d["python"]</p>
+$# $else:
+$#     <form method=post action=/sites/machines/$machine["name"]/builds>
+$#     <input type=hidden name=program value=python>
+$#     <div class=buttons><button>Install Python</button></div>
+$#     </form>
+$# 
+$# <hr>
+$# 
+$# $if d.get("canopy"):
+$#     <p>Canopy: $d["canopy"]</p>
+$#     <h4>Identities</h4>
+$#     $# <ul id=domains>
+$#     $# $for domain in domains:
+$#     $#     <li><a href=/sites/domains/$domain["name"]>$domain["name"]</a><br>
+$#     $#     <small>$domain</small></li>
+$#     $# </ul>
+$#     <form method=post>
+$#     <label><small>Address</small><br>
+$#     <label class=bounding><input type=radio> <input type=text
+$#     name=subdomain>&thinsp;.&thinsp;<select name=name>
+$#     $# $for domain, expiration in domains:
+$#     $#     <option value="$domain">$domain</option>
+$#     </select></label></label>
+$#     <label class=bounding><input type=radio> onion</label>
+$#     <div class=buttons><button>Add Identity</button></div>
+$#     </form>
+$# $else:
+$#     <form method=post action=/sites/machines/$machine["name"]/builds>
+$#     <input type=hidden name=program value=canopy>
+$#     <div class=buttons><button>Install Canopy</button></div>
+$#     </form>

+$def with (machines)
+$var title: Machines
+
+$machines