This commit is contained in:
Your Name
2025-05-25 11:03:49 +12:00
parent 161606a86a
commit 89bdd04ff8
5 changed files with 95 additions and 55 deletions

View File

@@ -42,15 +42,10 @@ Write actions:
- to upload a file (via http put) - to upload a file (via http put)
``` ```
curl -X PUT \ curl -X PUT \
-H "Content-Type: application/json" \ -H "Authorization: Bearer YOUR_TOKEN" \
-d '{ -F "file=@/path/to/your/file.txt" \
"label": "example", -F 'metadata={"labeltag":"example:latest","description":"Example file","tags":["test","example"],"custom_field":"custom value"}' \
"filename": "example.txt", "http://localhost:8123/upload"
"description": "Example file",
"tags": ["test", "example"],
"custom_field": "custom value"
}' \
"http://localhost:8123/upload?token=YOUR_TOKEN"
``` ```
- the object_file is uploaded, hashed, added to the registry (if that hash doesn't already exist), and {label:tag,hash} entries are added to the directory index. - the object_file is uploaded, hashed, added to the registry (if that hash doesn't already exist), and {label:tag,hash} entries are added to the directory index.
- matching tags on older versions are removed. - matching tags on older versions are removed.

View File

@@ -51,6 +51,33 @@ bool Server::init_db() {
bool Server::validate_write_request(const httplib::Request &req, httplib::Response &res, const std::vector<std::string> &required_params, std::map<std::string, std::string> &params) bool Server::validate_write_request(const httplib::Request &req, httplib::Response &res, const std::vector<std::string> &required_params, std::map<std::string, std::string> &params)
{ {
// Get token from Authorization header
std::string token;
if (req.has_header("Authorization")) {
const auto& auth_header = req.get_header_value("Authorization");
// Check if it's a Bearer token
if (auth_header.substr(0, 7) == "Bearer ") {
token = auth_header.substr(7);
}
}
if (token.empty()) {
res.status = 401;
nlohmann::json response = {{"result", "error"}, {"error", "Missing or invalid Authorization header"}};
res.set_content(response.dump(), "application/json");
return false;
}
// Check if token is valid
bool write_token_valid = std::find(config_.write_tokens.begin(), config_.write_tokens.end(), token) != config_.write_tokens.end();
if (!write_token_valid) {
res.status = 403;
nlohmann::json response = {{"result", "error"}, {"error", "Invalid write token"}};
res.set_content(response.dump(), "application/json");
return false;
}
// Get other parameters from query params
for (const auto& param : req.params) { for (const auto& param : req.params) {
params[param.first] = param.second; params[param.first] = param.second;
} }
@@ -65,15 +92,6 @@ bool Server::validate_write_request(const httplib::Request &req, httplib::Respon
} }
} }
// check token is valid
bool write_token_valid = std::find(config_.write_tokens.begin(), config_.write_tokens.end(), params["token"]) != config_.write_tokens.end();
if (!write_token_valid) {
res.status = 403;
nlohmann::json response = {{"result", "error"}, {"error", "Invalid write token"}};
res.set_content(response.dump(), "application/json");
return false;
}
return true; return true;
} }
@@ -281,7 +299,7 @@ void Server::handle_put_object(const httplib::Request& req, httplib::Response& r
// Check all request parameters first before processing any data // Check all request parameters first before processing any data
std::map<std::string, std::string> params; std::map<std::string, std::string> params;
if (!validate_write_request(req, res, {"token", "labeltag", "filename"}, params)) { if (!validate_write_request(req, res, {}, params)) { // No required params now since token is in header
return; return;
} }
@@ -293,7 +311,42 @@ void Server::handle_put_object(const httplib::Request& req, httplib::Response& r
return; return;
} }
auto [label, tag] = parse_label_tag(params["labeltag"]); // Parse the multipart form data
if (!req.has_file("file")) {
res.status = 400;
nlohmann::json response = {{"result", "error"}, {"error", "No file provided in upload"}};
res.set_content(response.dump(), "application/json");
return;
}
// Get the file data
const auto& file = req.get_file_value("file");
// Parse metadata if provided
nlohmann::json metadata;
if (req.has_file("metadata")) {
try {
const auto& metadata_file = req.get_file_value("metadata");
metadata = nlohmann::json::parse(metadata_file.content);
} catch (const nlohmann::json::parse_error& e) {
res.status = 400;
nlohmann::json response = {{"result", "error"}, {"error", "Invalid JSON metadata: " + std::string(e.what())}};
res.set_content(response.dump(), "application/json");
return;
}
}
// Validate required metadata fields
if (!metadata.contains("labeltag")) {
res.status = 400;
nlohmann::json response = {{"result", "error"}, {"error", "Missing required metadata field: labeltag"}};
res.set_content(response.dump(), "application/json");
return;
}
// Extract labeltag and validate format
std::string labeltag = metadata["labeltag"];
auto [label, tag] = parse_label_tag(labeltag);
if (label.empty() || tag.empty()) { if (label.empty() || tag.empty()) {
res.status = 400; res.status = 400;
nlohmann::json response = {{"result", "error"}, {"error", "Invalid label:tag format"}}; nlohmann::json response = {{"result", "error"}, {"error", "Invalid label:tag format"}};
@@ -301,6 +354,11 @@ void Server::handle_put_object(const httplib::Request& req, httplib::Response& r
return; return;
} }
// Add filename to metadata if not provided
if (!metadata.contains("filename")) {
metadata["filename"] = file.filename;
}
// Now that all parameters are validated, process the upload // Now that all parameters are validated, process the upload
// Generate a random number for the temporary filename // Generate a random number for the temporary filename
@@ -319,29 +377,14 @@ void Server::handle_put_object(const httplib::Request& req, httplib::Response& r
return; return;
} }
// Write request body to temporary file in chunks to handle large files better // Write file content to temporary file
// This improves memory usage even though the entire body is still loaded if (!temp_file.write(file.content.c_str(), file.content.size())) {
// by httplib - a proper streaming solution would require changes to httplib res.status = 500;
const size_t CHUNK_SIZE = 1024 * 1024; // 1MB chunks nlohmann::json response = {{"result", "error"}, {"error", "Failed to write to temporary file"}};
size_t remaining = req.body.size(); res.set_content(response.dump(), "application/json");
size_t offset = 0; temp_file.close();
std::filesystem::remove(temp_path);
while (remaining > 0) { return;
size_t write_size = std::min(CHUNK_SIZE, remaining);
if (!temp_file.write(req.body.data() + offset, write_size)) {
res.status = 500;
nlohmann::json response = {{"result", "error"}, {"error", "Failed to write to temporary file"}};
res.set_content(response.dump(), "application/json");
temp_file.close();
std::filesystem::remove(temp_path);
return;
}
offset += write_size;
remaining -= write_size;
// Periodically flush to disk
temp_file.flush();
} }
temp_file.close(); temp_file.close();
@@ -358,20 +401,14 @@ void Server::handle_put_object(const httplib::Request& req, httplib::Response& r
return; return;
} }
nlohmann::json metadata; // Add file metadata
add_file_metadata(temp_path.string(), metadata);
// Check for filename query parameter
std::string filename = "";
if (req.has_param("filename"))
metadata["original_filename"] = params["filename"];
// Check if filename ends with ".tgz" using the utility function // Check if filename ends with ".tgz" using the utility function
if (utils::ends_with(params["filename"], ".tgz")) { if (utils::ends_with(metadata["filename"], ".tgz")) {
metadata["tgz_content_hash"] = get_hash_from_tgz(temp_path.string()); metadata["tgz_content_hash"] = get_hash_from_tgz(temp_path.string());
} }
add_file_metadata(temp_path.string(), metadata);
// Move file to final location // Move file to final location
std::filesystem::path final_path = config_.object_store_path / std::to_string(hash); std::filesystem::path final_path = config_.object_store_path / std::to_string(hash);
if (!std::filesystem::exists(final_path)) { if (!std::filesystem::exists(final_path)) {
@@ -389,9 +426,9 @@ void Server::handle_put_object(const httplib::Request& req, httplib::Response& r
// Update database index // Update database index
dbEntry entry; dbEntry entry;
entry.label_tag = params["labeltag"]; entry.label_tag = labeltag;
entry.hash = std::to_string(hash); entry.hash = std::to_string(hash);
entry.metadata = metadata; // Store the potentially updated metadata entry.metadata = metadata; // Store the complete metadata
if (!db_->update_or_insert(entry)) { if (!db_->update_or_insert(entry)) {
res.status = 500; res.status = 500;

View File

@@ -53,7 +53,13 @@ BASE_TAG="autotest"
# upload this script as an object # upload this script as an object
echo "uploading ${SCRIPT_DIR}/${SCRIPT_NAME} to ${BASE_TAG}:test1" echo "uploading ${SCRIPT_DIR}/${SCRIPT_NAME} to ${BASE_TAG}:test1"
OBJECT_HASH=$(curl -s "${BASE_URL}/upload?token=${WRITE_TOKEN}&labeltag=${BASE_TAG}:test1&filename=${SCRIPT_NAME}" -T ${SCRIPT_DIR}/${SCRIPT_NAME} | jq -r '.hash') OBJECT_HASH=$(curl -X PUT \
-H "Authorization: Bearer ${WRITE_TOKEN}" \
-F "file=@${SCRIPT_DIR}/${SCRIPT_NAME}" \
-F 'metadata={"labeltag":"${BASE_TAG}:test1","description":"Example file","tags":["test","example"],"custom_field":"custom value"}' \
"http://localhost:8123/upload" | jq -r '.hash')
#OBJECT_HASH=$(curl -s "${BASE_URL}/upload?token=${WRITE_TOKEN}&labeltag=${BASE_TAG}:test1&filename=${SCRIPT_NAME}" -T ${SCRIPT_DIR}/${SCRIPT_NAME} | jq -r '.hash')
echo "received hash ${OBJECT_HASH}" echo "received hash ${OBJECT_HASH}"
# check the hash matches. # check the hash matches.

1
test.sh.downloaded1 Normal file
View File

@@ -0,0 +1 @@
{"error":"Invalid hash: null","result":"error"}

1
test.sh.downloaded2 Normal file
View File

@@ -0,0 +1 @@
{"error":"Invalid hash: autotest:test1","result":"error"}