Registry

README.md

@@ -0,0 +1,29 @@
+# Dropshell Template Registry
+
+## Introduction
+
+The DropShell template registry is a very simple C++ webserver 
+which provide a store of binary objects (the objects can be large),
+which are available over http.
+
+Read access is public.
+Write access is controlled by tokens.
+
+- Objects are access via a label and tag, or via their hash. For example:
+   - `wget http://dtr.jde.nz/object/squashkiwi:latest`
+   - `wget http://dtr.jde.nz/object/4528400792837739857`
+- The hash is calculated using `uint64_t hash_file(const std::string &path);` in hash.hpp.
+- You can retrieve the hash for a given labvel and tag with, e.g.:
+   - `curl http://dtr.jde.nz/hash/squashkiwi:latest`
+- you can get a full list of {label:tag,hash} entries (one tag per entry) with:
+   - `curl http://dtr.jde.nz/dir`
+- a simple welcome page is served at `/index.html` for those browsing to the site.
+- to upload a file (via http put)
+   - `curl -T object_file http://dtr.jde.nz/WRITE_TOKEN/LABEL:TAG`
+   - the object_file is uploaded, hashed, added to the registry (if that hash doesn't already exist), and {label:tag,hash} is added to the directory index.
+- the server is configured via a configuration file which allows setting:
+   - the list of write access tokens
+   - the location for the object store (path on disk)
+
+The C++ program is written so that it only uses standard libraries, and is built as a static executable with musl not glibc.
+

hash.cpp

@@ -0,0 +1,115 @@
+#include "utils/hash.hpp"
+
+#define XXH_INLINE_ALL
+#include "contrib/xxhash.hpp"
+
+#include <fstream>
+#include <filesystem>
+#include <iostream>
+
+namespace dropshell {
+
+uint64_t hash_file(const std::string &path) {
+    // Create hash state
+    XXH64_state_t* const state = XXH64_createState();
+    if (state == nullptr) {
+        std::cerr << "Failed to create hash state" << std::endl;
+        return 0;
+    }
+
+    // Initialize state with seed 0
+    XXH64_hash_t const seed = 0;   /* or any other value */
+    if (XXH64_reset(state, seed) == XXH_ERROR) return 0;
+
+    // Open file
+    std::ifstream file(path, std::ios::binary);
+    if (!file.is_open()) {
+        std::cerr << "Failed to open file: " << path << std::endl;
+        XXH64_freeState(state);
+        return 0;
+    }
+
+    // Read file in chunks and update hash
+    const size_t buffer_size = 4096;
+    char buffer[buffer_size];
+    while (file.read(buffer, buffer_size)) {
+        if (XXH64_update(state, buffer, file.gcount()) == XXH_ERROR) {
+            std::cerr << "Failed to update hash" << std::endl;
+            XXH64_freeState(state);
+            return 0;
+        }
+    }
+
+    // Handle any remaining bytes
+    if (file.gcount() > 0) {
+        if (XXH64_update(state, buffer, file.gcount()) == XXH_ERROR) {
+            std::cerr << "Failed to update hash" << std::endl;
+            XXH64_freeState(state);
+            return 0;
+        }
+    }
+
+    // Get final hash
+    XXH64_hash_t hash = XXH64_digest(state);
+    XXH64_freeState(state);
+    return hash;
+}
+
+uint64_t hash_directory_recursive(const std::string &path) {
+    // Create hash state
+    XXH64_state_t* const state = XXH64_createState();
+    if (state == nullptr) {
+        std::cerr << "Failed to create hash state" << std::endl;
+        return 0;
+    }
+
+    // Initialize state with seed 0
+    XXH64_hash_t const seed = 0;   /* or any other value */
+    if (XXH64_reset(state, seed) == XXH_ERROR) {
+        std::cerr << "Failed to reset hash state" << std::endl;
+        XXH64_freeState(state);
+        return 0;
+    }
+
+    try {
+        // Iterate through all files in directory recursively
+        for (const auto& entry : std::filesystem::recursive_directory_iterator(path)) {
+            if (entry.is_regular_file()) {
+                // Get file hash
+                XXH64_hash_t file_hash = hash_file(entry.path().string());
+                XXH64_update(state, &file_hash, sizeof(file_hash));
+            }
+        }
+    } catch (const std::filesystem::filesystem_error& e) {
+        std::cerr << "Filesystem error: " << e.what() << std::endl;
+        XXH64_freeState(state);
+        return 0;
+    }
+
+    // Get final hash
+    XXH64_hash_t hash = XXH64_digest(state);
+    XXH64_freeState(state);
+    return hash;
+}
+
+void hash_demo(const std::string & path)
+{
+    std::cout << "Hashing directory: " << path << std::endl;
+    auto start = std::chrono::high_resolution_clock::now();
+    XXH64_hash_t hash = hash_directory_recursive(path);
+    auto end = std::chrono::high_resolution_clock::now();
+    auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(end - start);
+    std::cout << "Hash: " << hash << " (took " << duration.count() << "ms)" << std::endl;
+}
+
+int hash_demo_raw(const std::string & path)
+{
+    if (!std::filesystem::exists(path)) {
+        std::cout << 0 <<std::endl; return 1;
+    }
+    XXH64_hash_t hash = hash_directory_recursive(path);
+    std::cout << hash << std::endl;
+    return 0;
+}
+
+} // namespace dropshell

hash.hpp

@@ -0,0 +1,20 @@
+#ifndef HASH_HPP
+#define HASH_HPP
+
+#include <string>
+#include <cstdint>
+
+namespace dropshell {
+
+    uint64_t hash_file(const std::string &path);
+
+    uint64_t hash_directory_recursive(const std::string &path);
+
+    void hash_demo(const std::string & path);
+
+    int hash_demo_raw(const std::string & path);
+
+} // namespace dropshell
+
+
+#endif