public/simple-object-server
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Bug fixing

Browse Source
This commit is contained in:
Your Name
2025-05-25 13:07:03 +12:00
parent 21d079f285
commit 7b3706ebf5
5 changed files with 68 additions and 65 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
README.md
View File

@@ -4,7 +4,7 @@ A simple object storage system that stores files with metadata and provides a RE
## Features
- Store files with metadata (labels, tags, and custom fields)
- Store files with metadata (label:tag pairs and custom fields)
- Retrieve files by hash or label:tag combination
- Check if a file exists by hash or label:tag
- Delete files by hash
@@ -47,8 +47,7 @@ PUT /upload
Parameters:
- `file`: The file to upload
- `metadata`: JSON object containing:
- `labels`: Array of strings (required)
- `tags`: Array of strings (required)
- `labeltags`: Array of strings in "label:tag" format (required)
- Additional custom fields (optional)
Example:
@@ -56,7 +55,7 @@ Example:
curl -X PUT \
-H "Authorization: Bearer your-token" \
-F "file=@example.txt" \
-F 'metadata={"labels":["test"],"tags":["latest"],"description":"Example file"}' \
-F 'metadata={"labeltags":["test:latest","project:alpha"],"description":"Example file"}' \
http://localhost:8080/upload
```
@@ -115,9 +114,8 @@ The system uses SQLite to store metadata about uploaded files. The database sche
```sql
CREATE TABLE objects (
hash TEXT PRIMARY KEY,
labels TEXT NOT NULL, -- JSON array of labels
tags TEXT NOT NULL, -- JSON array of tags
metadata TEXT NOT NULL -- JSON object with additional metadata
labeltags TEXT NOT NULL, -- JSON array of label:tag pairs
metadata TEXT NOT NULL -- JSON object with additional metadata
);
```

62
src/database.cpp
View File

@@ -11,7 +11,7 @@ bool Database::createObjectsTable() {
const char* create_table_sql =
"CREATE TABLE IF NOT EXISTS objects ("
"hash TEXT PRIMARY KEY,"
"label_tags TEXT NOT NULL," // JSON array of label:tag pairs
"labeltags TEXT NOT NULL," // JSON array of label:tag pairs
"metadata TEXT NOT NULL"
");";
@@ -186,10 +186,10 @@ bool Database::get(const std::string& key, dbEntry& entry) {
std::string sql;
if (key.find(':') != std::string::npos) {
// Query by label:tag
sql = "SELECT hash, label_tags, metadata FROM objects WHERE label_tags LIKE ?;";
sql = "SELECT hash, labeltags, metadata FROM objects WHERE labeltags LIKE ?;";
} else {
// Query by hash
sql = "SELECT hash, label_tags, metadata FROM objects WHERE hash = ?;";
sql = "SELECT hash, labeltags, metadata FROM objects WHERE hash = ?;";
}
sqlite3_stmt* stmt;
@@ -211,10 +211,10 @@ bool Database::get(const std::string& key, dbEntry& entry) {
}
entry.hash = reinterpret_cast<const char*>(sqlite3_column_text(stmt, 0));
std::string label_tags_str = reinterpret_cast<const char*>(sqlite3_column_text(stmt, 1));
std::string labeltags_str = reinterpret_cast<const char*>(sqlite3_column_text(stmt, 1));
std::string metadata_str = reinterpret_cast<const char*>(sqlite3_column_text(stmt, 2));
entry.label_tags = nlohmann::json::parse(label_tags_str).get<std::vector<std::string>>();
entry.label_tags = nlohmann::json::parse(labeltags_str).get<std::vector<std::string>>();
entry.metadata = nlohmann::json::parse(metadata_str);
sqlite3_finalize(stmt);
@@ -222,7 +222,7 @@ bool Database::get(const std::string& key, dbEntry& entry) {
}
bool Database::list(std::vector<dbEntry>& entries) {
std::string sql = "SELECT hash, label_tags, metadata FROM objects;";
std::string sql = "SELECT hash, labeltags, metadata FROM objects;";
sqlite3_stmt* stmt;
if (sqlite3_prepare_v2(db_, sql.c_str(), -1, &stmt, nullptr) != SQLITE_OK) {
@@ -233,10 +233,10 @@ bool Database::list(std::vector<dbEntry>& entries) {
while (sqlite3_step(stmt) == SQLITE_ROW) {
dbEntry entry;
entry.hash = reinterpret_cast<const char*>(sqlite3_column_text(stmt, 0));
std::string label_tags_str = reinterpret_cast<const char*>(sqlite3_column_text(stmt, 1));
std::string labeltags_str = reinterpret_cast<const char*>(sqlite3_column_text(stmt, 1));
std::string metadata_str = reinterpret_cast<const char*>(sqlite3_column_text(stmt, 2));
entry.label_tags = nlohmann::json::parse(label_tags_str).get<std::vector<std::string>>();
entry.label_tags = nlohmann::json::parse(labeltags_str).get<std::vector<std::string>>();
entry.metadata = nlohmann::json::parse(metadata_str);
entries.push_back(entry);
}
@@ -247,12 +247,12 @@ bool Database::list(std::vector<dbEntry>& entries) {
bool Database::merge_existing_entry(const dbEntry& existing, const dbEntry& new_entry, dbEntry& merged) {
// Merge label:tag pairs
std::set<std::string> merged_label_tags(existing.label_tags.begin(), existing.label_tags.end());
merged_label_tags.insert(new_entry.label_tags.begin(), new_entry.label_tags.end());
std::set<std::string> merged_labeltags(existing.label_tags.begin(), existing.label_tags.end());
merged_labeltags.insert(new_entry.label_tags.begin(), new_entry.label_tags.end());
// Create merged entry
merged = new_entry; // Start with new entry's data
merged.label_tags = std::vector<std::string>(merged_label_tags.begin(), merged_label_tags.end());
merged.label_tags = std::vector<std::string>(merged_labeltags.begin(), merged_labeltags.end());
// Update metadata - preserve fields from existing entry that aren't in new entry
merged.metadata = existing.metadata; // Start with existing metadata
@@ -261,21 +261,21 @@ bool Database::merge_existing_entry(const dbEntry& existing, const dbEntry& new_
}
// Ensure required fields are set correctly
merged.metadata["label_tags"] = merged.label_tags;
merged.metadata["labeltags"] = merged.label_tags;
merged.metadata["hash"] = merged.hash;
// Update database
std::string sql = "UPDATE objects SET label_tags = ?, metadata = ? WHERE hash = ?;";
std::string sql = "UPDATE objects SET labeltags = ?, metadata = ? WHERE hash = ?;";
sqlite3_stmt* stmt;
if (sqlite3_prepare_v2(db_, sql.c_str(), -1, &stmt, nullptr) != SQLITE_OK) {
return false;
}
std::string label_tags_str = nlohmann::json(merged.label_tags).dump();
std::string labeltags_str = nlohmann::json(merged.label_tags).dump();
std::string metadata_str = merged.metadata.dump();
sqlite3_bind_text(stmt, 1, label_tags_str.c_str(), -1, SQLITE_STATIC);
sqlite3_bind_text(stmt, 1, labeltags_str.c_str(), -1, SQLITE_STATIC);
sqlite3_bind_text(stmt, 2, metadata_str.c_str(), -1, SQLITE_STATIC);
sqlite3_bind_text(stmt, 3, merged.hash.c_str(), -1, SQLITE_STATIC);
@@ -285,23 +285,23 @@ bool Database::merge_existing_entry(const dbEntry& existing, const dbEntry& new_
}
bool Database::insert_new_entry(const dbEntry& entry) {
std::string sql = "INSERT INTO objects (hash, label_tags, metadata) VALUES (?, ?, ?);";
std::string sql = "INSERT INTO objects (hash, labeltags, metadata) VALUES (?, ?, ?);";
sqlite3_stmt* stmt;
if (sqlite3_prepare_v2(db_, sql.c_str(), -1, &stmt, nullptr) != SQLITE_OK) {
return false;
}
// Update metadata to include label_tags and hash
// Update metadata to include labeltags and hash
nlohmann::json metadata = entry.metadata;
metadata["label_tags"] = entry.label_tags;
metadata["labeltags"] = entry.label_tags;
metadata["hash"] = entry.hash;
std::string label_tags_str = nlohmann::json(entry.label_tags).dump();
std::string labeltags_str = nlohmann::json(entry.label_tags).dump();
std::string metadata_str = metadata.dump();
sqlite3_bind_text(stmt, 1, entry.hash.c_str(), -1, SQLITE_STATIC);
sqlite3_bind_text(stmt, 2, label_tags_str.c_str(), -1, SQLITE_STATIC);
sqlite3_bind_text(stmt, 2, labeltags_str.c_str(), -1, SQLITE_STATIC);
sqlite3_bind_text(stmt, 3, metadata_str.c_str(), -1, SQLITE_STATIC);
bool success = sqlite3_step(stmt) == SQLITE_DONE;
@@ -312,7 +312,7 @@ bool Database::insert_new_entry(const dbEntry& entry) {
bool Database::handle_tag_conflicts(const dbEntry& entry) {
for (const auto& label_tag : entry.label_tags) {
// Find all entries with this exact label:tag pair
std::string find_sql = "SELECT hash, label_tags, metadata FROM objects WHERE label_tags LIKE ?;";
std::string find_sql = "SELECT hash, labeltags, metadata FROM objects WHERE labeltags LIKE ?;";
sqlite3_stmt* stmt;
if (sqlite3_prepare_v2(db_, find_sql.c_str(), -1, &stmt, nullptr) != SQLITE_OK) {
return false;
@@ -325,39 +325,39 @@ bool Database::handle_tag_conflicts(const dbEntry& entry) {
std::string other_hash = reinterpret_cast<const char*>(sqlite3_column_text(stmt, 0));
if (other_hash == entry.hash) continue; // Skip our own entry
std::string other_label_tags_str = reinterpret_cast<const char*>(sqlite3_column_text(stmt, 1));
std::string other_labeltags_str = reinterpret_cast<const char*>(sqlite3_column_text(stmt, 1));
std::string other_metadata_str = reinterpret_cast<const char*>(sqlite3_column_text(stmt, 2));
// Parse the other entry
dbEntry other;
other.hash = other_hash;
other.label_tags = nlohmann::json::parse(other_label_tags_str).get<std::vector<std::string>>();
other.label_tags = nlohmann::json::parse(other_labeltags_str).get<std::vector<std::string>>();
other.metadata = nlohmann::json::parse(other_metadata_str);
// Remove the exact label:tag pair
std::vector<std::string> new_label_tags;
std::vector<std::string> new_labeltags;
for (const auto& other_label_tag : other.label_tags) {
if (other_label_tag != label_tag) {
new_label_tags.push_back(other_label_tag);
new_labeltags.push_back(other_label_tag);
}
}
// Update the other entry if it had the label:tag pair removed
if (new_label_tags.size() != other.label_tags.size()) {
other.label_tags = new_label_tags;
other.metadata["label_tags"] = new_label_tags; // Update metadata to match
if (new_labeltags.size() != other.label_tags.size()) {
other.label_tags = new_labeltags;
other.metadata["labeltags"] = new_labeltags; // Update metadata to match
std::string update_sql = "UPDATE objects SET label_tags = ?, metadata = ? WHERE hash = ?;";
std::string update_sql = "UPDATE objects SET labeltags = ?, metadata = ? WHERE hash = ?;";
sqlite3_stmt* update_stmt;
if (sqlite3_prepare_v2(db_, update_sql.c_str(), -1, &update_stmt, nullptr) != SQLITE_OK) {
sqlite3_finalize(stmt);
return false;
}
std::string new_label_tags_str = nlohmann::json(new_label_tags).dump();
std::string new_labeltags_str = nlohmann::json(new_labeltags).dump();
std::string new_metadata_str = other.metadata.dump();
sqlite3_bind_text(update_stmt, 1, new_label_tags_str.c_str(), -1, SQLITE_STATIC);
sqlite3_bind_text(update_stmt, 1, new_labeltags_str.c_str(), -1, SQLITE_STATIC);
sqlite3_bind_text(update_stmt, 2, new_metadata_str.c_str(), -1, SQLITE_STATIC);
sqlite3_bind_text(update_stmt, 3, other.hash.c_str(), -1, SQLITE_STATIC);

27
src/put_handler.cpp
View File

@@ -67,18 +67,28 @@ void PutHandler::handle_put_object(const httplib::Request& req, httplib::Respons
}
// Validate required metadata fields
if (!metadata.contains("labels") || !metadata["labels"].is_array() || metadata["labels"].empty()) {
if (!metadata.contains("labeltags") || !metadata["labeltags"].is_array() || metadata["labeltags"].empty()) {
res.status = 400;
nlohmann::json response = {{"result", "error"}, {"error", "Missing or invalid required metadata field: labels (must be non-empty array)"}};
nlohmann::json response = {{"result", "error"}, {"error", "Missing or invalid required metadata field: labeltags (must be non-empty array of label:tag pairs)"}};
res.set_content(response.dump(), "application/json");
return;
}
if (!metadata.contains("tags") || !metadata["tags"].is_array() || metadata["tags"].empty()) {
res.status = 400;
nlohmann::json response = {{"result", "error"}, {"error", "Missing or invalid required metadata field: tags (must be non-empty array)"}};
res.set_content(response.dump(), "application/json");
return;
// Validate each label:tag pair format
for (const auto& label_tag : metadata["labeltags"]) {
if (!label_tag.is_string()) {
res.status = 400;
nlohmann::json response = {{"result", "error"}, {"error", "Invalid label:tag pair format - must be a string"}};
res.set_content(response.dump(), "application/json");
return;
}
std::string pair = label_tag.get<std::string>();
if (pair.find(':') == std::string::npos) {
res.status = 400;
nlohmann::json response = {{"result", "error"}, {"error", "Invalid label:tag pair format - must contain ':' separator"}};
res.set_content(response.dump(), "application/json");
return;
}
}
// Add filename to metadata if not provided
@@ -155,8 +165,7 @@ void PutHandler::handle_put_object(const httplib::Request& req, httplib::Respons
// Update database index
dbEntry entry;
entry.hash = std::to_string(hash);
entry.labels = metadata["labels"].get<std::vector<std::string>>();
entry.tags = metadata["tags"].get<std::vector<std::string>>();
entry.label_tags = metadata["labeltags"].get<std::vector<std::string>>();
entry.metadata = metadata;
if (!server_.db_->update_or_insert(entry)) {

30
test.sh
View File

@@ -62,8 +62,7 @@ echo "Simple Object Storage server is running at ${BASE_URL}"
# Construct metadata JSON
METADATA_JSON=$(cat <<EOF
{
"labels": ["${BASE_TAG}"],
"tags": ["test1"],
"labeltags": ["${BASE_TAG}:test1"],
"description": "Example file",
"custom_field": "custom value"
}
@@ -133,8 +132,7 @@ title "Testing metadata field preservation"
# Upload with extra metadata fields
EXTRA_METADATA_JSON=$(cat <<EOF
{
"labels": ["${BASE_TAG}"],
"tags": ["test2"],
"labeltags": ["${BASE_TAG}:test2"],
"description": "Test with extra fields",
"custom_field": "custom value",
"extra_field1": "value1",
@@ -200,8 +198,7 @@ title "Testing tag versioning behavior"
# Upload first version with tag 'latest'
FIRST_METADATA_JSON=$(cat <<EOF
{
"labels": ["${BASE_TAG}"],
"tags": ["latest", "v1"],
"labeltags": ["${BASE_TAG}:latest", "${BASE_TAG}:v1"],
"description": "First version"
}
EOF
@@ -217,17 +214,16 @@ UPLOAD_RESPONSE=$(curl -X PUT \
FIRST_HASH=$(echo ${UPLOAD_RESPONSE} | jq -r '.hash')
# Store first version's metadata before uploading second version
FIRST_METADATA=$(curl -s "${BASE_URL}/meta/${BASE_TAG}:latest")
FIRST_METADATA=$(curl -s "${BASE_URL}/meta/${FIRST_HASH}")
echo "First version metadata response: ${FIRST_METADATA}"
if ! echo "${FIRST_METADATA}" | jq -r '.metadata.tags[]' | grep -q 'v1'; then
if ! echo "${FIRST_METADATA}" | jq -r '.metadata.labeltags[]' | grep -q "${BASE_TAG}:v1"; then
die "First version does not have v1 tag"
fi
# Upload second version with same tag 'latest'
SECOND_METADATA_JSON=$(cat <<EOF
{
"labels": ["${BASE_TAG}"],
"tags": ["latest", "v2"],
"labeltags": ["${BASE_TAG}:latest", "${BASE_TAG}:v2"],
"description": "Second version"
}
EOF
@@ -249,24 +245,24 @@ SECOND_HASH=$(echo ${UPLOAD_RESPONSE} | jq -r '.hash')
# Verify first version's metadata still has v1 tag
FIRST_METADATA=$(curl -s "${BASE_URL}/meta/${FIRST_HASH}")
echo "First version metadata response: ${FIRST_METADATA}"
if ! echo "${FIRST_METADATA}" | jq -r '.metadata.tags[]' | grep -q 'v1'; then
if ! echo "${FIRST_METADATA}" | jq -r '.metadata.labeltags[]' | grep -q "${BASE_TAG}:v1"; then
die "First version does not have v1 tag"
fi
# Verify first version's metadata no longer has the latest tag
if echo "${FIRST_METADATA}" | jq -r '.metadata.tags[]' | grep -q 'latest'; then
if echo "${FIRST_METADATA}" | jq -r '.metadata.labeltags[]' | grep -q "${BASE_TAG}:latest"; then
die "First version still has latest tag"
fi
# Verify second version has the correct tags: v2 and latest
SECOND_METADATA=$(curl -s "${BASE_URL}/meta/${BASE_TAG}:latest")
SECOND_METADATA=$(curl -s "${BASE_URL}/meta/${SECOND_HASH}")
echo "Second version metadata response: ${SECOND_METADATA}"
if ! echo "${SECOND_METADATA}" | jq -r '.metadata.tags[]' | grep -q 'v2'; then
die "Second version does not have v2 tag"
fi
if ! echo "${SECOND_METADATA}" | jq -r '.metadata.tags[]' | grep -q 'latest'; then
if ! echo "${SECOND_METADATA}" | jq -r '.metadata.labeltags[]' | grep -q "${BASE_TAG}:latest"; then
die "Second version does not have latest tag"
fi
if ! echo "${SECOND_METADATA}" | jq -r '.metadata.labeltags[]' | grep -q "${BASE_TAG}:v2"; then
die "Second version does not have v2 tag"
fi
# Clean up
curl -s -H "Authorization: Bearer ${WRITE_TOKEN}" "${BASE_URL}/deleteobject?hash=${FIRST_HASH}" > /dev/null

2
test_1GB_file_upload.sh
View File

@@ -29,7 +29,7 @@ echo "Uploading file..."
RESPONSE=$(curl -X PUT \
-H "Authorization: Bearer ${WRITE_TOKEN}" \
-F "file=@test_file.bin" \
-F 'metadata={"labels":["test"],"tags":["latest","large"],"description":"Test file"}' \
-F 'metadata={"labeltags":["test:latest","test:large"],"description":"Test file"}' \
"http://${HOST}:${PORT}/upload")
echo "Upload response: $RESPONSE"