external & build dependences in src tree

- external & build dependences boost licensed
  - ext_depends (external depends)
    - D-YAML
      - tinyendian
    - d2sqlite3
    - imageformats
  - build_depends
    - dub2nix

12 files changed, 606 insertions, 0 deletions

diff --git a/src/build_depends/dub2nix/.github/workflows/main.yml b/src/build_depends/dub2nix/.github/workflows/main.yml
new file mode 100644
index 0000000..fe9e10a
--- /dev/null
+++ b/src/build_depends/dub2nix/.github/workflows/main.yml
@@ -0,0 +1,31 @@
+# This is a basic workflow to help you get started with Actions
+
+name: CI
+
+# Controls when the action will run. Triggers the workflow on push or pull request
+# events but only for the master branch
+on:
+  push:
+    branches: [ master ]
+  pull_request:
+    branches: [ master ]
+
+# A workflow run is made up of one or more jobs that can run sequentially or in parallel
+jobs:
+  # This workflow contains a single job called "build"
+  build:
+    # The type of runner that the job will run on
+    runs-on: ubuntu-latest
+
+    # Steps represent a sequence of tasks that will be executed as part of the job
+    steps:
+    # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
+    - uses: actions/checkout@v2
+
+    - name: Install Nix
+      uses: cachix/install-nix-action@v12
+      with:
+        nix_path: nixpkgs=channel:nixos-unstable
+      
+    - name: Build in shell
+      run: nix-shell --run "dub test"
diff --git a/src/build_depends/dub2nix/.gitignore b/src/build_depends/dub2nix/.gitignore
new file mode 100644
index 0000000..19eefee
--- /dev/null
+++ b/src/build_depends/dub2nix/.gitignore
@@ -0,0 +1,5 @@
+dub2nix
+/.dub/
+/result
+/bin
+.envrc
diff --git a/src/build_depends/dub2nix/LICENSE b/src/build_depends/dub2nix/LICENSE
new file mode 100644
index 0000000..88e0035
--- /dev/null
+++ b/src/build_depends/dub2nix/LICENSE
@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2020 Lionello Lunesu
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
diff --git a/src/build_depends/dub2nix/README.md b/src/build_depends/dub2nix/README.md
new file mode 100644
index 0000000..bfb8e3a
--- /dev/null
+++ b/src/build_depends/dub2nix/README.md
@@ -0,0 +1,82 @@
+# dub2nix
+CLI tool to create Nix expressions for D-lang Dub projects.
+
+![CI](https://github.com/lionello/dub2nix/workflows/CI/badge.svg)
+
+## Installation
+Install with `nix-env`:
+```sh
+nix-env -if https://github.com/lionello/dub2nix/archive/master.tar.gz
+```
+or add to your `shell.nix` and run `nix-shell`:
+```nix
+with import <nixpkgs> {};
+let
+  dub2nix-src = fetchTarball {
+    url = "https://github.com/lionello/dub2nix/archive/master.tar.gz";
+  };
+  dub2nix = (import dub2nix-src) { inherit pkgs; };
+in mkShell {
+  buildInputs = [
+    dub2nix # dub dmd rdmd ldc etc..
+  ];
+}
+```
+
+## Development
+Do `git clone` and `nix-shell` to build with `dub`:
+```sh
+nix-shell
+dub
+```
+Alternatively, use `direnv`:
+```sh
+echo use nix >> .envrc
+direnv allow
+dub
+```
+
+## Usage
+```
+Usage: dub2nix [OPTIONS] COMMAND
+
+Create Nix derivations for Dub package dependencies.
+
+Commands:
+  save         Write Nix files for Dub project
+
+Options:
+-i     --input Path of selections JSON; defaults to ./dub.selections.json
+-o    --output Output Nix file for Dub project
+-r  --registry URL to Dub package registry; default http://code.dlang.org/packages/
+-d --deps-file Output Nix file with dependencies; defaults to ./dub.selections.nix
+-h      --help This help information.
+```
+First, use `dub build` to generate the `dub.selections.json` for your Dub project.
+Then, run `dub2nix save` to read the `dub.selections.json` in the current folder and write a new file `dub.selections.nix`.
+
+This `dub.selections.nix` is used in `mkDubDerivation` (from `mkDub.nix`) to create a new derivation for your Dub project:
+```nix
+{pkgs}:
+with (import ./mkDub.nix {
+    inherit pkgs;
+});
+mkDubDerivation {
+    version = "0.1.0"; # optional
+    src = ./.;
+}
+```
+
+When your project has no dependencies at all, this will fail because `dub.selections.nix` is missing. Set `deps` to override the dependencies:
+```nix
+{pkgs}:
+with (import ./mkDub.nix {
+    inherit pkgs;
+});
+mkDubDerivation {
+    src = ./.;
+    deps = [];
+}
+```
+
+Use the `--output` option to write a `.nix` file with a skeleton derivation for your dub project. This will also create the `mkDub.nix` file for importing into the derivation.
diff --git a/src/build_depends/dub2nix/default.nix b/src/build_depends/dub2nix/default.nix
new file mode 100644
index 0000000..d967504
--- /dev/null
+++ b/src/build_depends/dub2nix/default.nix
@@ -0,0 +1,2 @@
+{ pkgs ? import <nixpkgs> {} }:
+pkgs.callPackage ./dub2nix.nix {}
diff --git a/src/build_depends/dub2nix/dub.json b/src/build_depends/dub2nix/dub.json
new file mode 100644
index 0000000..18ed81d
--- /dev/null
+++ b/src/build_depends/dub2nix/dub.json
@@ -0,0 +1,15 @@
+{
+  "name": "dub2nix",
+  "authors": [
+    "Lionello Lunesu"
+  ],
+  "dependencies": {
+    "vibe-d:data": "*"
+  },
+  "stringImportPaths": ["."],
+  "description": "Create Nix derivations for Dub package dependencies",
+  "targetPath": "./bin/",
+  "copyright": "Copyright © 2020, Lionello Lunesu",
+  "license": "MIT",
+  "targetType": "executable"
+}
diff --git a/src/build_depends/dub2nix/dub.selections.json b/src/build_depends/dub2nix/dub.selections.json
new file mode 100644
index 0000000..de6fe4d
--- /dev/null
+++ b/src/build_depends/dub2nix/dub.selections.json
@@ -0,0 +1,7 @@
+{
+	"fileVersion": 1,
+	"versions": {
+		"stdx-allocator": "2.77.5",
+		"vibe-d": "0.8.5"
+	}
+}
diff --git a/src/build_depends/dub2nix/dub.selections.nix b/src/build_depends/dub2nix/dub.selections.nix
new file mode 100644
index 0000000..26a9c05
--- /dev/null
+++ b/src/build_depends/dub2nix/dub.selections.nix
@@ -0,0 +1,26 @@
+# This file was generated by https://github.com/lionello/dub2nix v0.2.4
+[ {
+  fetch = {
+    type = "git";
+    url = "https://github.com/vibe-d/vibe.d.git";
+    rev = "v0.8.5";
+    sha256 = "0s1caxqmq2497j5x8h06f44nr597h9zac8qxxml953lkaqkhbzgy";
+    fetchSubmodules = false;
+    date = "2019-03-24T14:45:15+01:00";
+    deepClone = false;
+    leaveDotGit = false;
+    path = "/nix/store/kz5g44dncvznlkm38a74cmm4qyl9nr6b-vibe.d";
+  };
+} {
+  fetch = {
+    type = "git";
+    url = "https://github.com/wilzbach/stdx-allocator.git";
+    rev = "v2.77.5";
+    sha256 = "03av8zp5p6vf6fg005xbmwnjfw96jyrr4dcj4m56c4a3vx7v72pk";
+    fetchSubmodules = false;
+    date = "2018-12-23T13:54:22+01:00";
+    deepClone = false;
+    leaveDotGit = false;
+    path = "/nix/store/b3h25asfh205wzwjfzjf2k2kkccpp96k-stdx-allocator";
+  };
+} ]
\ No newline at end of file
diff --git a/src/build_depends/dub2nix/dub2nix.nix b/src/build_depends/dub2nix/dub2nix.nix
new file mode 100644
index 0000000..ef4bd5f
--- /dev/null
+++ b/src/build_depends/dub2nix/dub2nix.nix
@@ -0,0 +1,17 @@
+{pkgs}:
+with (import ./mkDub.nix {
+    inherit pkgs;
+});
+mkDubDerivation {
+    src = pkgs.lib.cleanSource ./.;
+    # dubJSON = ./dub.json;
+    # selections = ./dub.selections.nix;
+    version = "0.2.4";
+    # doCheck = true;
+    propagatedBuildInputs = [ pkgs.nix-prefetch-git pkgs.cacert ];
+    meta = with pkgs.stdenv.lib; {
+       homepage = "https://github.com/lionello/dub2nix";
+       maintainers = [ maintainers.lionello ];
+       license = licenses.mit;
+    };
+}
diff --git a/src/build_depends/dub2nix/mkDub.nix b/src/build_depends/dub2nix/mkDub.nix
new file mode 100644
index 0000000..7fddf14
--- /dev/null
+++ b/src/build_depends/dub2nix/mkDub.nix
@@ -0,0 +1,121 @@
+{ pkgs ? import <nixpkgs> {},
+  stdenv ? pkgs.stdenv,
+  rdmd ? pkgs.rdmd,
+  dmd ? pkgs.dmd,
+  dub ? pkgs.dub }:
+
+with stdenv;
+let
+  # Filter function to remove the .dub package folder from src
+  filterDub = name: type: let baseName = baseNameOf (toString name); in ! (
+    type == "directory" && baseName == ".dub"
+  );
+
+  # Convert a GIT rev string (tag) to a simple semver version
+  rev-to-version = builtins.replaceStrings ["v" "refs/tags/v"] ["" ""];
+
+  dep2src = dubDep: pkgs.fetchgit { inherit (dubDep.fetch) url rev sha256 fetchSubmodules; };
+
+  # Fetch a dependency (source only for now)
+  fromDub = dubDep: mkDerivation rec {
+    name = "${src.name}-${version}";
+    version = rev-to-version dubDep.fetch.rev;
+    nativeBuildInputs = [ rdmd dmd dub ];
+    src = dep2src dubDep;
+
+    buildPhase = ''
+      runHook preBuild
+      export HOME=$PWD
+      dub build -b=release
+      runHook postBuild
+    '';
+
+    # outputs = [ "lib" ];
+
+    # installPhase = ''
+    #   runHook preInstall
+    #   mkdir -p $out/bin
+    #   runHook postInstall
+    # '';
+  };
+
+  # Adds a local package directory (e.g. a git repository) to Dub
+  dub-add-local = dubDep: "dub add-local ${(fromDub dubDep).src.outPath} ${rev-to-version dubDep.fetch.rev}";
+
+  # The target output of the Dub package
+  targetOf = package: "${package.targetPath or "."}/${package.targetName or package.name}";
+
+  # Remove reference to build tools and library sources
+  disallowedReferences = deps: [ dmd rdmd dub ] ++ builtins.map dep2src deps;
+
+  removeExpr = refs: ''remove-references-to ${lib.concatMapStrings (ref: " -t ${ref}") refs}'';
+
+in {
+  inherit fromDub;
+
+  mkDubDerivation = lib.makeOverridable ({
+    src,
+    nativeBuildInputs ? [],
+    dubJSON ? src + "/dub.json",
+    selections ? src + "/dub.selections.nix",
+    deps ? import selections,
+    passthru ? {},
+    package ? lib.importJSON dubJSON,
+    ...
+  } @ attrs: stdenv.mkDerivation ((removeAttrs attrs ["package" "deps" "selections" "dubJSON"]) // {
+
+    pname = package.name;
+
+    nativeBuildInputs = [ rdmd dmd dub pkgs.removeReferencesTo ] ++ nativeBuildInputs;
+    disallowedReferences = disallowedReferences deps;
+
+    passthru = passthru // {
+      inherit dub dmd rdmd pkgs;
+    };
+
+    src = lib.cleanSourceWith {
+      filter = filterDub;
+      src = lib.cleanSource src;
+    };
+
+    preFixup = ''
+      find $out/bin -type f -exec ${removeExpr (disallowedReferences deps)} '{}' + || true
+    '';
+
+    buildPhase = ''
+      runHook preBuild
+
+      export HOME=$PWD
+      ${lib.concatMapStringsSep "\n" dub-add-local deps}
+      dub build -b release --combined --skip-registry=all
+
+      runHook postBuild
+    '';
+
+    checkPhase = ''
+      runHook preCheck
+
+      export HOME=$PWD
+      ${lib.concatMapStringsSep "\n" dub-add-local deps}
+      dub test --combined --skip-registry=all
+
+      runHook postCheck
+    '';
+
+    installPhase = ''
+      runHook preInstall
+
+      mkdir -p $out/bin
+      cp -r "${targetOf package}" $out/bin
+
+      runHook postInstall
+    '';
+
+    meta = lib.optionalAttrs (package ? description) {
+      description = package.description;
+    } // attrs.meta or {};
+  } // lib.optionalAttrs (!(attrs ? version)) {
+    # Use name from dub.json, unless pname and version are specified
+    name = package.name;
+  }));
+}
diff --git a/src/build_depends/dub2nix/shell.nix b/src/build_depends/dub2nix/shell.nix
new file mode 100644
index 0000000..002ff55
--- /dev/null
+++ b/src/build_depends/dub2nix/shell.nix
@@ -0,0 +1,16 @@
+with import <nixpkgs> {};
+
+let
+  pkg = import ./default.nix { inherit pkgs; };
+
+in mkShell {
+
+  buildInputs = [
+    # additional runtime dependencies go here
+  ] ++ pkg.buildInputs ++ pkg.propagatedBuildInputs;
+
+  nativeBuildInputs = [
+    # additional dev dependencies go here
+  ] ++ pkg.nativeBuildInputs;
+
+}
diff --git a/src/build_depends/dub2nix/src/dub2nix.d b/src/build_depends/dub2nix/src/dub2nix.d
new file mode 100644
index 0000000..e92ecf8
--- /dev/null
+++ b/src/build_depends/dub2nix/src/dub2nix.d
@@ -0,0 +1,263 @@
+#!/usr/bin/env dub
+/+ dub.sdl:
+    name "dub2nix"
+    stringImportPaths "."
+    dependency "vibe-d:data" version="*"
++/
+import vibe.data.json, std.string;
+
+enum mkDubNix = import("./mkDub.nix");
+enum VERSION = "0.2.4";
+enum HEADER = "# This file was generated by https://github.com/lionello/dub2nix v"~VERSION~"\n";
+
+unittest {
+    static assert(import("./dub2nix.nix").indexOf(VERSION) > 0, "VERSION does not match version in dub2nix.nix");
+}
+
+struct DubSelections {
+    int fileVersion;
+    string[string] versions;
+}
+
+struct DubRepo {
+    string owner;
+    string kind;
+    string project;
+}
+
+private string packageRegistry = "http://code.dlang.org/packages/";
+
+private auto download(string url) @trusted {
+version(none) {
+    // This works, but causes "leaking eventcore driver" warnings at shutdown
+    import vibe.http.client : requestHTTP;
+    scope res = requestHTTP();
+    return res.readJson();
+} else {
+    import std.net.curl: get, HTTP;
+    auto http = HTTP();
+    // Using deflate saves A LOT of traffic, ~40x
+    http.addRequestHeader("accept-encoding", "deflate");
+    http.addRequestHeader("accept", "application/json");
+    const data = get(url, http);
+    // Only accepting application/json, so anything else must be compressed
+    if (data[0] != '{') {
+        import std.zlib : uncompress;
+        return parseJsonString(cast(string)uncompress(data));
+    } else {
+        // parseJsonString takes immutable string, so need the .idup here
+        return parseJsonString(data.idup);
+    }
+}
+}
+
+/// Query Dub registry for the repository information of a package
+auto findRepo(string pname) @safe {
+    const url = packageRegistry ~ pname ~ ".json";
+    const json = download(url);
+    return deserializeJson!DubRepo(json["repository"]);
+}
+
+struct NixPrefetchGit {
+    @optional string type;          /// set to "git", like Go deps.nix
+    string url;                     /// URL of GIT repository
+    string rev;                     /// sha1 or tag
+    string sha256;                  /// calculated by from nix-prefetch-git
+    @optional bool fetchSubmodules; /// optional; defaults to true
+    @optional string date;          /// ignored; fetchgit doesn't actually want this
+    @optional bool deepClone;       /// ignored
+    @optional bool leaveDotGit;     /// ignored; if the .git directory should be preserved
+    @optional string path;          /// ignored; a path in the Nix store?
+}
+
+/// Invoke nix-prefetch-git and return the parsed JSON
+auto nixPrefetchGit(string url, string rev) @safe {
+    import std.process : executeShell, Config;
+    const cmd = "nix-prefetch-git --quiet " ~ url ~ " " ~ rev;
+    return deserializeJson!NixPrefetchGit(
+        executeShell(cmd, null, Config.stderrPassThrough).output
+    );
+}
+
+struct DubDep {
+    NixPrefetchGit fetch;           /// like Go deps.nix
+}
+
+/// Fetch the repo information for package `pname` and version `ver`
+auto prefetch(string pname, string ver) @safe {
+    const repo = findRepo(pname);
+    assert(repo.kind == "github");
+    const url = "https://" ~ repo.kind ~ ".com/" ~ repo.owner ~ '/' ~ repo.project ~ ".git";
+    const tag = "v" ~ ver;
+    auto set = nixPrefetchGit(url, tag);
+    // Overwrite the sha1 ref with the tag instead, so we have the version info as well
+    set.rev = tag;
+    set.type = "git";
+    return DubDep(set);
+}
+
+/// Convert D string to Nix string literal
+auto toNixString(in string s, int indent = 0) pure @safe {
+    if (s is null) {
+        return "null";
+    } else if (s.indexOfAny("\"\n") >= 0)
+        return "''\n" ~ s ~ "''";
+    else
+        return '"' ~ s ~ '"';
+}
+
+unittest {
+    static assert(toNixString(null) == "null");
+    static assert(toNixString("hello") == `"hello"`);
+    static assert(toNixString("with\nnewline") == "''\nwith\nnewline''");
+    static assert(toNixString(`with "quotes"`) == "''\nwith \"quotes\"''");
+}
+
+/// Convert D bool to Nix boolean literal
+auto toNixString(bool b, int indent = 0) pure @safe {
+    return b ? "true" : "false";
+}
+
+unittest {
+    static assert(toNixString(true) == "true");
+    static assert(toNixString(false) == "false");
+}
+
+private enum INDENT = "                                                              ";
+
+/// Convert D struct to Nix set
+auto toNixString(T)(in T pod, int indent = 0) pure @safe if (is(T == struct)) {
+    string prefix = INDENT[0..indent * 2 + 2];
+    string set = "{\n";
+    foreach(i, ref key; pod.tupleof) {
+        const id = __traits(identifier, pod.tupleof[i]);
+        set ~= prefix ~ id ~ " = " ~ toNixString(key, indent + 1) ~ ";\n";
+    }
+    return set ~ INDENT[0..indent * 2] ~ "}";
+}
+
+unittest {
+    struct TestStruct { bool b; }
+    static assert(toNixString(TestStruct.init) == "{\n  b = false;\n}");
+    static assert(toNixString(TestStruct.init, 1) == "{\n    b = false;\n  }");
+}
+
+/// Convert D AArray to Nix set
+auto toNixString(T)(in T[string] aa, int indent = 0) pure @safe {
+    string prefix = INDENT[0..indent * 2 + 2];
+    string set = "{\n";
+    foreach(id, ref key; aa) {
+        set ~= prefix ~ id ~ " = " ~ toNixString(key, indent + 1) ~ ";\n";
+    }
+    return set ~ INDENT[0..indent * 2] ~ "}";
+}
+
+unittest {
+    static assert(toNixString(["s": "x"]) == "{\n  s = \"x\";\n}");
+    static assert(toNixString(["s": ["x": true]]) == "{\n  s = {\n    x = true;\n  };\n}");
+}
+
+/// Convert D array/range to Nix list
+import std.range : isForwardRange;
+auto toNixString(R)(in R range, int indent = 0) pure @safe if (isForwardRange!R && !is(R : string)) {
+    string list = "[ ";
+    foreach(const ref item; range) {
+        list ~= toNixString(item, indent) ~ " ";
+    }
+    return list ~ "]";
+}
+
+unittest {
+    static assert(toNixString(["a"]) == `[ "a" ]`);
+}
+
+/// Create Nix expression for all dependencies in the selections JSON
+auto createNixDeps(string selectionsJson) {
+    import std.parallelism : taskPool;
+    import std.array : byPair, array;
+    import std.stdio : writeln;
+
+    const selections = deserializeJson!DubSelections(selectionsJson);
+    assert(selections.fileVersion == 1);
+
+    static auto progress(Tuple)(in Tuple pair) {
+        writeln("# Prefetching ", pair.key, "-", pair.value);
+        return prefetch(pair.key, pair.value);
+    }
+
+    // Fetch all dependency information in parallel
+    debug scope(success) writeln("# Done.");
+    return HEADER ~ toNixString(taskPool.amap!progress(selections.versions.byPair.array));
+}
+
+unittest {
+    enum json = import("./dub.selections.json");
+    enum fixture = import("./dub.selections.nix");
+    assert(createNixDeps(json) == fixture);
+}
+
+// No "main" when we're running with unittests
+version(unittest) {} else {
+
+int main(string[] args) {
+    import std.stdio : writeln;
+    import std.file : readText, write;
+    import std.getopt: getopt, defaultGetoptPrinter;
+
+    bool showVersion;
+    string input = "./dub.selections.json", deps = "./dub.selections.nix", output;
+    auto result = getopt(args,
+        "input|i|in", "Path of selections JSON; defaults to " ~ input, &input,
+        "output|o|out", "Output Nix file for Dub project.", &output,
+        "registry|r", "URL to Dub package registry; default " ~ packageRegistry, &packageRegistry,
+        "deps-file|d", "Output Nix file with dependencies; defaults to " ~ deps, &deps,
+        "version", "Show version information.", &showVersion);
+
+    if (showVersion) {
+        writeln(VERSION);
+        return 0;
+    } else if (result.helpWanted || args.length != 2 || args[1] != "save") {
+        defaultGetoptPrinter(`Usage: dub2nix [OPTIONS] COMMAND
+
+Create Nix derivations for Dub package dependencies.
+
+Commands:
+  save         Write Nix files for Dub project
+
+Options:`, result.options);
+        return 0;
+    }
+
+    try {
+        const nix = createNixDeps(readText(input));
+        if (deps == "-") {
+            writeln(nix);
+        } else {
+            write(deps, nix.representation);
+        }
+        if (output) {
+            write("mkDub.nix", mkDubNix);
+            write(output, HEADER ~ `
+{ pkgs ? import <nixpkgs> {} }:
+with import ./mkDub.nix { inherit pkgs; };
+
+mkDubDerivation {
+    src = ./.;
+    # version = "0.0.1";
+    # buildInputs = [ add any runtime deps here ];
+}
+`);
+        }
+        return 0;
+    } catch (Exception e) {
+        debug {
+            // Only dump callstack in debug builds
+            writeln(e.toString());
+        } else {
+            writeln("Error: ", e.message);
+        }
+        return 1;
+    }
+}
+
+}//!unittest