diff --git a/.gitignore b/.gitignore index 1066b5c..3c81c91 100644 --- a/.gitignore +++ b/.gitignore @@ -1,11 +1,15 @@ .DS_Store __pycache__ +*.bak # config config/server.json config/*.bu config/*.ign +# minecraft +minecraft/server.properties + diff --git a/README.md b/README.md index b10f490..b5135e2 100644 --- a/README.md +++ b/README.md @@ -22,8 +22,8 @@ Services deployed on [matthewtran.com](https://matthewtran.com). ## update quick dev => scp dockerfiles => rebuild locally -final dev => reprovision - +final dev => reprovision + wipe home so images rebuilds +logs => sudo -u game podman logs container TODO fix setup_router DUID suff => may need to reset after each provision... diff --git a/compose.yml b/compose.yml deleted file mode 100644 index 5a58a0e..0000000 --- a/compose.yml +++ /dev/null @@ -1,134 +0,0 @@ -networks: - web: - enable_ipv6: true - ipam: - config: - - subnet: "172.20.0.0/16" - - subnet: "fd3a:138e:8fd0:0020::/64" - monero: - enable_ipv6: true - ipam: - config: - - subnet: "172.21.0.0/16" - - subnet: "fd3a:138e:8fd0:0021::/64" - game: - enable_ipv6: true - ipam: - config: - - subnet: "172.22.0.0/16" - - subnet: "fd3a:138e:8fd0:0022::/64" - nas: - enable_ipv6: true - ipam: - config: - - subnet: "172.23.0.0/16" - - subnet: "fd3a:138e:8fd0:0023::/64" -services: - website: - restart: always - build: website/. - entrypoint: ["/bin/sh", "/home/me/entry.sh"] - ports: - - "80:8080" - - "443:8443" - networks: - - web - volumes: - - ./website/certbot:/home/me/certbot - cap_drop: - - ALL - gitea: - restart: always - image: gitea/gitea:latest-rootless - user: "2000:2000" - ports: - - "2222:2222" - networks: - - web - volumes: - - ./website/gitea/data:/var/lib/gitea - - ./website/gitea/config:/etc/gitea - - /etc/timezone:/etc/timezone:ro - - /etc/localtime:/etc/localtime:ro - cap_drop: - - ALL - monerod: - restart: always - build: monerod/. - entrypoint: ["/bin/sh", "/home/me/entry.sh"] - stdin_open: true - tty: true - ports: - - "18080:18080" - - "18081:18081" - networks: - - monero - volumes: - - ./monerod/.bitmonero:/home/me/.bitmonero - cap_drop: - - ALL - p2pool: - stop_grace_period: 1m # TODO reduce m_shutdownCountdown to reduce this - restart: always - build: p2pool/. - entrypoint: ["/bin/sh", "/home/me/entry.sh"] - stdin_open: true - tty: true - ports: - - "3333:3333" - - "37888:37888" - - "37889:37889" - networks: - - monero - volumes: - - ./p2pool/cache:/home/me/cache - cap_drop: - - ALL - minecraft: - restart: always - build: minecraft/. - entrypoint: ["/bin/sh", "/home/me/entry.sh"] - ports: - - "25565:25565" - networks: - - game - volumes: - - ./minecraft/worlds:/home/me/worlds - cap_drop: - - ALL - # minecraft_bedrock: - # restart: always - # build: minecraft_bedrock/. - # entrypoint: ["/bin/sh", "/home/me/entry.sh"] - # ports: - # - "19132:19132/udp" - # - "19133:19133/udp" - # networks: - # - game - # volumes: - # - ./minecraft_bedrock/worlds:/home/me/worlds - # cap_drop: - # - ALL - terraria: - restart: always - build: terraria/. - entrypoint: ["/usr/bin/python3", "/home/me/entry.py"] - ports: - - "7777:7777" - networks: - - game - volumes: - - ./terraria/worlds:/home/me/worlds - - ./terraria/mods:/home/me/mods - cap_drop: - - ALL - nas: - restart: always - build: nas/. - entrypoint: ["/bin/sh", "/home/me/entry.sh"] - ports: - - "445:8445" - networks: - - nas - cap_drop: - - ALL diff --git a/config/server.default b/config/server.default index 4bf665c..a935d35 100644 --- a/config/server.default +++ b/config/server.default @@ -4,8 +4,9 @@ "ssh_keys": [ "ssh-ed25519 AAAA..." ], - "var_key": "", - "var_wipe": false + "home_key": "", + "home_wipe": false, + "data_dir": "/var/home/core/matthewtrancom_data" }, "drives": [ { @@ -14,5 +15,8 @@ "name": "stuff", "wipe": false } - ] + ], + "minecraft": { + "world": "main" + } } \ No newline at end of file diff --git a/minecraft/.dockerignore b/minecraft/.dockerignore deleted file mode 100644 index 6a2fdbd..0000000 --- a/minecraft/.dockerignore +++ /dev/null @@ -1 +0,0 @@ -worlds/ diff --git a/minecraft/Dockerfile b/minecraft/Dockerfile index f9605dd..d761d0f 100644 --- a/minecraft/Dockerfile +++ b/minecraft/Dockerfile @@ -3,18 +3,18 @@ FROM ubuntu:24.04 RUN apt-get update && apt-get -y upgrade RUN apt-get install -y wget openjdk-21-jre -RUN groupadd -g 2002 me && useradd -u 2002 -g 2002 -m me -USER me -WORKDIR /home/me +WORKDIR /root # from https://github.com/itzg/rcon-cli -RUN wget -O rcon-cli.tar.gz https://github.com/itzg/rcon-cli/releases/download/1.6.9/rcon-cli_1.6.9_linux_amd64.tar.gz +RUN wget -O rcon-cli.tar.gz https://github.com/itzg/rcon-cli/releases/download/1.7.0/rcon-cli_1.7.0_linux_amd64.tar.gz RUN tar xvf rcon-cli.tar.gz && rm rcon-cli.tar.gz -# from https://www.minecraft.net/en-us/download/server (currently 1.21.4) -RUN wget https://piston-data.mojang.com/v1/objects/4707d00eb834b446575d89a61a11b5d548d8c001/server.jar +# from https://www.minecraft.net/en-us/download/server (currently 1.21.5) +RUN wget https://piston-data.mojang.com/v1/objects/e6ec2f64e6080b9b5d9b471b291c33cc7f509733/server.jar -COPY --chown=me:me eula.txt ./ -COPY --chown=me:me entry.sh ./ -COPY --chown=me:me server.properties ./ -COPY --chown=me:me ops.json ./ +COPY entry.sh ./ +COPY eula.txt ./ +COPY ops.json ./ +COPY server.properties ./ + +CMD ["/bin/bash", "/root/entry.sh"] diff --git a/minecraft/entry.sh b/minecraft/entry.sh index 14eeb2d..fab019d 100644 --- a/minecraft/entry.sh +++ b/minecraft/entry.sh @@ -4,7 +4,7 @@ cleanup() { ./rcon-cli --password password stop } -trap 'cleanup' TERM +trap 'cleanup' SIGTERM SIGINT java -Xmx1024M -Xms1024M -jar server.jar nogui & wait $! # wait for SIGTERM diff --git a/minecraft/server.properties b/minecraft/server.default similarity index 98% rename from minecraft/server.properties rename to minecraft/server.default index 99bb9e6..569c3cc 100644 --- a/minecraft/server.properties +++ b/minecraft/server.default @@ -20,7 +20,6 @@ hardcore=true hide-online-players=false initial-disabled-packs= initial-enabled-packs=vanilla -level-name=worlds/main level-seed= level-type=minecraft\:normal max-chained-neighbor-updates=1000000 diff --git a/scripts/provision.py b/scripts/provision.py index 0a042cd..024a639 100755 --- a/scripts/provision.py +++ b/scripts/provision.py @@ -5,24 +5,32 @@ import json import secrets import subprocess import yaml +from pathlib import Path +from update import SOURCE_DIR, IMAGES, generate -if __name__ == "__main__": - cfg = json.load(open("config/server.json")) - but = { - "variant": "fcos", - "version": "1.6.0", - } +UIDS = { + "web" : 1001, + "monero" : 1002, + "game" : 1003, + "nas" : 1004, +} - # recommend keys if needed - if "var_key" not in cfg["core"]: - print(f'cfg["core"]["var_key"] doesn\'t exist, try "{base64.b64encode(secrets.token_bytes(64)).decode("utf-8")}"') +PORTS = { + "game": [ + "25565:25565", + ], +} + +def check_keys(): + if "home_key" not in cfg["core"]: + print(f'cfg["core"]["home_key"] doesn\'t exist, try "{base64.b64encode(secrets.token_bytes(64)).decode("utf-8")}"') exit(1) for i, d in enumerate(cfg["drives"]): if "key" not in d: print(f'cfg["drives"][{i}]["key"] doesn\'t exist, try "{base64.b64encode(secrets.token_bytes(64)).decode("utf-8")}"') exit(1) - # configure root drive +def add_root_drive(): but["storage"] = { "disks": [ { @@ -36,7 +44,7 @@ if __name__ == "__main__": "resize": True, }, { - "label": "var", + "label": "home", "size_mib": 0, }, ], @@ -52,10 +60,10 @@ if __name__ == "__main__": "clevis": { "tpm2": True }, }, { - "name": "var", - "device": "/dev/disk/by-partlabel/var", - "wipe_volume": cfg["core"]["var_wipe"], - "key_file": { "inline": base64.b64decode(cfg["core"]["var_key"]) }, + "name": "home", + "device": "/dev/disk/by-partlabel/home", + "wipe_volume": cfg["core"]["home_wipe"], + "key_file": { "inline": base64.b64decode(cfg["core"]["home_key"]) }, }, ], "filesystems": [ @@ -66,18 +74,18 @@ if __name__ == "__main__": "label": "root", }, { - "path": "/var", - "device": "/dev/mapper/var", + "path": "/var/home", + "device": "/dev/mapper/home", "format": "xfs", - "wipe_filesystem": cfg["core"]["var_wipe"], + "wipe_filesystem": cfg["core"]["home_wipe"], "with_mount_unit": True, }, ], - "files": [], "directories": [], + "files": [], } - # add additional drives +def add_more_drive(): for d in cfg["drives"]: raid = len(d["devices"]) > 1 if raid: @@ -105,18 +113,7 @@ if __name__ == "__main__": "group": { "name": "core" }, }) - # add SSH keys - assert(len(cfg["core"]["ssh_keys"]) > 0) - but["passwd"] = { - "users": [ - { - "name": "core", - "ssh_authorized_keys": cfg["core"]["ssh_keys"], - }, - ], - } - - # add packages +def add_packages(): # TODO update once done https://github.com/coreos/fedora-coreos-tracker/issues/681 but["systemd"] = { "units": [ @@ -147,30 +144,150 @@ if __name__ == "__main__": ], } - # set hostname +def add_ssh_keys(): + assert(len(cfg["core"]["ssh_keys"]) > 0) + but["passwd"] = { + "users": [ + { + "name": "core", + "ssh_authorized_keys": cfg["core"]["ssh_keys"], + }, + ], + } + +def set_hostname(): but["storage"]["files"].append({ "path": "/etc/hostname", "mode": 0o644, "contents": { "inline": cfg["core"]["hostname"] }, }) - # allow unprivileged port access +def allow_port_access(): but["storage"]["files"].append({ "path": "/etc/sysctl.d/99-unprivileged-ports.conf", "mode": 0o644, "contents": { "inline": "net.ipv4.ip_unprivileged_port_start=80" }, }) +def add_users(): + for user in UIDS: + but["passwd"]["users"].append({ + "name": user, + "uid": UIDS[user], + }) + but["storage"]["files"].append({ + "path": f"/var/lib/systemd/linger/{user}", + "contents": { "inline": "" }, + }) - # TODO make server build images on first boot? - # TODO serve backup.zip to restore on first boot? only if wipe specified +def copy_source(): + but["storage"]["directories"].append({ + "path": SOURCE_DIR, + "user": { "name": "core" }, + "group": { "name": "core" }, + }) + for i in (f for s in IMAGES.values() for f in s): + but["storage"]["directories"].append({ + "path": str(Path(SOURCE_DIR) / i), + "user": { "name": "core" }, + "group": { "name": "core" }, + }) + for f in Path(i).glob("*"): + but["storage"]["files"].append({ + "path": str(Path(SOURCE_DIR) / f), + "contents": { "inline": open(f, "r").read() }, + "user": { "name": "core" }, + "group": { "name": "core" }, + }) - # TODO convert all to quadlets? whatever compose likes +def build_images(): + but["storage"]["directories"].append({ "path": "/etc/containers/systemd/users" }) + for user in IMAGES: + but["storage"]["directories"].append({ "path": f"/etc/containers/systemd/users/{UIDS[user]}" }) + for img in IMAGES[user]: + but["storage"]["files"].append({ + "path": f"/etc/containers/systemd/users/{UIDS[user]}/{img}.build", + "contents": { "inline": "\n".join([ + "[Build]", + f"ImageTag={img}", + f"SetWorkingDirectory={SOURCE_DIR}/{img}", + ])} + }) + +def create_pods(): + for user in IMAGES: + but["storage"]["files"].append({ + "path": f"/etc/containers/systemd/users/{UIDS[user]}/{user}.pod", + "contents": { "inline": "[Pod]\n" + "\n".join([f"PublishPort={p}" for p in PORTS[user]])} + }) + +def create_folders(): + but["storage"]["directories"].append({ + "path": cfg["core"]["data_dir"], + "user": { "name": "core" }, + "group": { "name": "core" }, + }) + for user in IMAGES: + for img in IMAGES[user]: + but["storage"]["directories"].append({ + "path": str(Path(cfg["core"]["data_dir"]) / img), + "user": { "name": user }, + "group": { "name": user }, + }) + +def run_containers(): + for user in IMAGES: + for img in IMAGES[user]: + but["storage"]["files"].append({ + "path": f"/etc/containers/systemd/users/{UIDS[user]}/{img}.container", + "contents": { "inline": "\n".join([ + "[Container]", + f"ContainerName={img}", + f"Image={img}.build", + f"Pod={user}.pod", + f"Volume={str(Path(cfg["core"]["data_dir"]) / img)}:/root/data:z", + "[Install]", + "WantedBy=default.target", + ])} + }) + +if __name__ == "__main__": + cfg = json.load(open("config/server.json")) + but = { + "variant": "fcos", + "version": "1.6.0", + } + + # core setup + check_keys() + add_root_drive() + add_more_drive() + add_packages() + add_ssh_keys() + set_hostname() + allow_port_access() + + # server setup + add_users() + generate(cfg) + copy_source() + build_images() + create_pods() + create_folders() + run_containers() + + + # TODO add rest of containers + # add core to nas group + # TODO script to backup => restore backup if desired # TODO enable bedrock => check idle cpu # TODO reduce disk logging? + # TODO generate ISO, else nginx if --insecure with open("config/server.bu", "w") as f: f.write(yaml.dump(but, sort_keys=False)) subprocess.check_output(["butane", "-p", "-s", "-o", "config/server.ign", "config/server.bu"]) + + print("NOTE - TPM may need to be cleared after enough provisions.") print("WARNING - Using unencrypted connections without authentication, ensure LAN is secure!") diff --git a/scripts/update.py b/scripts/update.py new file mode 100755 index 0000000..f3fab87 --- /dev/null +++ b/scripts/update.py @@ -0,0 +1,55 @@ +#!/usr/bin/env python3 + +import json +import shutil +import subprocess +from pathlib import Path + +SOURCE_DIR = "/var/source" + +IMAGES = { + "game": [ + "minecraft", + ], +} + +def generate(cfg): + # minecraft + shutil.copy("minecraft/server.default", "minecraft/server.properties") + with open("minecraft/server.properties", "a") as f: + f.write(f"level-name=data/{cfg["minecraft"]["world"]}") + +def run(cmds): + try: + subprocess.check_output(["ssh", f"core@{cfg["core"]["hostname"]}.local", ";".join(cmds)], stderr=subprocess.STDOUT) + except subprocess.CalledProcessError as e: + print("\033[31m", end="") + print(e.output.decode()) + print("\033[0m", end="") + exit(1) + +if __name__ == "__main__": + cfg = json.load(open("config/server.json")) + + # generate helper files + generate(cfg) + + # copy files + for f in (f for l in IMAGES.values() for f in l): + subprocess.run(["scp", "-r", f, f"core@{cfg["core"]["hostname"]}.local:{SOURCE_DIR}"], check=True) + + # run builds + for user in IMAGES: + print(f"building images for {user}...") + run([f"cd {SOURCE_DIR}"] + [ + f"sudo -u {user} podman build --tag {i} {SOURCE_DIR}/{i}" + for i in IMAGES[user] + ]) + + # restart pods + for user in IMAGES: + print(f"restarting pod for {user}...") + run([ + f"cd {SOURCE_DIR}", + f"sudo systemctl --machine={user}@.host --user restart {user}-pod " + " ".join(IMAGES[user]), + ])