From a7ece306cda6ff158832e39ed9920f2aba9a252a Mon Sep 17 00:00:00 2001
From: Your Name <j@842.be>
Date: Sat, 3 May 2025 15:58:38 +1200
Subject: [PATCH] Big.

---
 src/server.cpp          | 38 +++++++++++++++++++++++++---
 test_1GB_file_upload.sh | 56 +++++++++++++++++++++++++++++++++++++++++
 2 files changed, 90 insertions(+), 4 deletions(-)
 create mode 100755 test_1GB_file_upload.sh

diff --git a/src/server.cpp b/src/server.cpp
index 5a414a8..1c76ff8 100644
--- a/src/server.cpp
+++ b/src/server.cpp
@@ -153,11 +153,18 @@ void Server::setup_routes() {
         handle_get_directory(req, res);
     });
 
-    // Upload object
-    server_.Put("/(.*)", [this](const httplib::Request& req, httplib::Response& res) {
+    // Upload object with streaming support
+    server_.Put("/upload", [this](const httplib::Request& req, httplib::Response& res) {
         handle_put_object(req, res);
     });
 
+    // Handle PUT requests to other paths
+    server_.Put("/(.*)", [this](const httplib::Request& req, httplib::Response& res) {
+        res.status = 404;
+        nlohmann::json response = {{"result", "error"}, {"error", "Not found - put requests must be to /upload"}};
+        res.set_content(response.dump(), "application/json");
+    });
+
     // Get metadata for label:tag
     server_.Get("/meta/(.*)", [this](const httplib::Request& req, httplib::Response& res) {
         handle_get_metadata(req, res);
@@ -310,8 +317,31 @@ void Server::handle_put_object(const httplib::Request& req, httplib::Response& r
         return;
     }
 
-    // Write request body to temporary file
-    temp_file.write(req.body.data(), req.body.size());
+    // Write request body to temporary file in chunks to handle large files better
+    // This improves memory usage even though the entire body is still loaded 
+    // by httplib - a proper streaming solution would require changes to httplib
+    const size_t CHUNK_SIZE = 1024 * 1024; // 1MB chunks
+    size_t remaining = req.body.size();
+    size_t offset = 0;
+    
+    while (remaining > 0) {
+        size_t write_size = std::min(CHUNK_SIZE, remaining);
+        if (!temp_file.write(req.body.data() + offset, write_size)) {
+            res.status = 500;
+            nlohmann::json response = {{"result", "error"}, {"error", "Failed to write to temporary file"}};
+            res.set_content(response.dump(), "application/json");
+            temp_file.close();
+            std::filesystem::remove(temp_path);
+            return;
+        }
+        
+        offset += write_size;
+        remaining -= write_size;
+        
+        // Periodically flush to disk
+        temp_file.flush();
+    }
+    
     temp_file.close();
 
     // Ensure the temporary file is removed even if errors occur
diff --git a/test_1GB_file_upload.sh b/test_1GB_file_upload.sh
new file mode 100755
index 0000000..8e4fa45
--- /dev/null
+++ b/test_1GB_file_upload.sh
@@ -0,0 +1,56 @@
+#!/bin/bash
+
+# Create a temporary test file (100MB)
+echo "Creating test file..."
+dd if=/dev/urandom of=test_file.bin bs=1M count=1024
+
+
+# read ~/.config/simple_object_storage/config.json
+CONFIG_PATH="${HOME}/.config/simple_object_storage/config.json"
+if [ ! -f "${CONFIG_PATH}" ]; then
+    echo "config file not found at ${CONFIG_PATH}"
+    exit 1
+fi
+CONFIG=$(cat "${CONFIG_PATH}")
+
+# get the host and port from the config
+HOST=$(echo $CONFIG | jq -r '.host')
+PORT=$(echo $CONFIG | jq -r '.port')
+
+# extract the first write token from the config
+WRITE_TOKEN=$(echo $CONFIG | jq -r '.write_tokens[0]')
+
+
+
+# Get a write token (assuming a write token 'test_token' is configured)
+TOKEN=${WRITE_TOKEN}
+
+# Upload the file
+echo "Uploading file..."
+RESPONSE=$(curl -X PUT -T test_file.bin "http://${HOST}:${PORT}/upload?token=${TOKEN}&labeltag=test:latest&filename=test_file.bin")
+echo "Upload response: $RESPONSE"
+
+# Extract the hash from the response
+HASH=$(echo $RESPONSE | jq -r '.hash')
+echo "Hash: $HASH"
+
+# Check if the file exists by hash
+echo "Checking if file exists by hash..."
+EXISTS_HASH=$(curl "http://${HOST}:${PORT}/exists/$HASH")
+echo "Exists by hash response: $EXISTS_HASH"
+
+# Check if the file exists by label:tag
+echo "Checking if file exists by label:tag..."
+EXISTS_TAG=$(curl "http://${HOST}:${PORT}/exists/test:latest")
+echo "Exists by label:tag response: $EXISTS_TAG"
+
+# delete the file!
+echo "Deleting file..."
+DELETE_RESPONSE=$(curl "http://${HOST}:${PORT}/deleteobject?token=${TOKEN}&hash=${HASH}")
+echo "Delete response: $DELETE_RESPONSE"
+
+# Clean up
+echo "Cleaning up..."
+rm test_file.bin
+
+echo "Test completed." 
\ No newline at end of file