public/simple-object-server
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Working

Browse Source
This commit is contained in:
Your Name
2025-05-03 15:08:25 +12:00
parent 7df5c75539
commit c456fb4225
6 changed files with 237 additions and 25 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
README.md
View File

@@ -2,7 +2,7 @@
## Introduction
The simple_object_storage template registry is a very simple C++ webserver
Simple Object Storage is a very simple C++ webserver
which provide a store of binary objects (the objects can be large),
which are available over http.
@@ -21,8 +21,14 @@ Write access is controlled by tokens.
- `curl http://localhost:8123/meta/squashkiwi:latest`
- a simple welcome page is served at `/index.html` for those browsing to the site.
- to upload a file (via http put)
- `curl -T object_file http://localhost:8123/WRITE_TOKEN/LABEL:TAG?filename="FILENAME"`
- `curl -T object_file http://localhost:8123/upload?token="WRITE_TOKEN"\&labeltag="LABEL:TAG"\&filename="FILENAME"`
- the object_file is uploaded, hashed, added to the registry (if that hash doesn't already exist), and {label:tag,hash} is added to the directory index.
- to delete a label/tag (object remains):
- `curl http://localhost:8123/deletetag?token="WRITE_TOKEN"\&labeltag="LABEL:TAG"`
- to delete an object (and all tags on that object):
- `curl http://localhost:8123/deleteobject?token="WRITE_TOKEN"\&hash="HASH"`
- add a tag to an existing object:
- `curl http://localhost:8123/appendtag?token="WRITE_TOKEN"\&hash="HASH"\&labeltag="LABEL:TAG"`
- the server is configured via a configuration file which allows setting:
- the list of write access tokens
- the location for the object store (path on disk)

14
src/database.cpp
View File

@@ -166,6 +166,20 @@ bool Database::remove(const std::string& label_tag) {
return success;
}
bool Database::remove_by_hash(const std::string& hash) {
std::string sql = "DELETE FROM objects WHERE hash = ?;";
sqlite3_stmt* stmt;
if (sqlite3_prepare_v2(db_, sql.c_str(), -1, &stmt, nullptr) != SQLITE_OK) {
return false;
}
sqlite3_bind_text(stmt, 1, hash.c_str(), -1, SQLITE_STATIC);
bool success = sqlite3_step(stmt) == SQLITE_DONE;
sqlite3_finalize(stmt);
return success;
}
bool Database::get(const std::string& label_tag, dbEntry& entry) {
std::string sql = "SELECT hash, metadata FROM objects WHERE label_tag = ?;";
sqlite3_stmt* stmt;

1
src/database.hpp
View File

@@ -24,6 +24,7 @@ class Database {
~Database();
bool insert(const dbEntry& entry);
bool remove(const std::string& label_tag);
bool remove_by_hash(const std::string& hash);
bool get(const std::string& label_tag, dbEntry& entry);
bool update(const std::string& label_tag, const dbEntry& entry);
bool list(std::vector<dbEntry>& entries);

190
src/server.cpp
View File

@@ -49,6 +49,32 @@ bool Server::init_db() {
}
}
bool Server::validate_write_request(const httplib::Request &req, httplib::Response &res, const std::vector<std::string> &required_params, std::map<std::string, std::string> &params)
{
for (const auto& param : req.params) {
params[param.first] = param.second;
}
// Check for required parameters
for (const auto& param : required_params) {
if (!req.has_param(param)) {
res.status = 400;
res.set_content("Missing required query parameter: " + param, "text/plain");
return false;
}
}
// check token is valid
bool write_token_valid = std::find(config_.write_tokens.begin(), config_.write_tokens.end(), params["token"]) != config_.write_tokens.end();
if (!write_token_valid) {
res.status = 403;
res.set_content("Invalid write token", "text/plain");
return false;
}
return true;
}
Server::Server(const ServerConfig& config)
: config_(config), running_(false) {
// Ensure object store directory exists
@@ -121,7 +147,7 @@ void Server::setup_routes() {
});
// Upload object
server_.Put("/([^/]+)/(.*)", [this](const httplib::Request& req, httplib::Response& res) {
server_.Put("/(.*)", [this](const httplib::Request& req, httplib::Response& res) {
handle_put_object(req, res);
});
@@ -129,6 +155,21 @@ void Server::setup_routes() {
server_.Get("/meta/(.*)", [this](const httplib::Request& req, httplib::Response& res) {
handle_get_metadata(req, res);
});
// Delete a label/tag (object remains)
server_.Get("/deletetag", [this](const httplib::Request& req, httplib::Response& res) {
handle_delete_tag(req, res);
});
// Delete an object (and all tags on that object)
server_.Get("/deleteobject", [this](const httplib::Request& req, httplib::Response& res) {
handle_delete_object(req, res);
});
// Add a tag to an existing object
server_.Get("/appendtag", [this](const httplib::Request& req, httplib::Response& res) {
handle_append_tag(req, res);
});
}
void Server::handle_get_object(const httplib::Request& req, httplib::Response& res) {
@@ -208,22 +249,29 @@ void Server::handle_get_directory(const httplib::Request& /*req*/, httplib::Resp
}
void Server::handle_put_object(const httplib::Request& req, httplib::Response& res) {
const auto& token = req.matches[1].str();
const auto& label_tag = req.matches[2].str();
if (!validate_write_token(token)) {
res.status = 403;
res.set_content("Invalid write token", "text/plain");
// Check all request parameters first before processing any data
std::map<std::string, std::string> params;
if (!validate_write_request(req, res, {"token", "labeltag", "filename"}, params)) {
return;
}
auto [label, tag] = parse_label_tag(label_tag);
// 1. Check we're in the /upload path
if (req.path != "/upload") {
res.status = 404;
res.set_content("Not found - put requests must be to /upload", "text/plain");
return;
}
auto [label, tag] = parse_label_tag(params["labeltag"]);
if (label.empty() || tag.empty()) {
res.status = 400;
res.set_content("Invalid label:tag format", "text/plain");
return;
}
// Now that all parameters are validated, process the upload
// Generate a random number for the temporary filename
std::mt19937_64 rng(std::chrono::high_resolution_clock::now().time_since_epoch().count());
std::uniform_int_distribution<uint64_t> dist;
@@ -258,21 +306,18 @@ void Server::handle_put_object(const httplib::Request& req, httplib::Response& r
// Check for filename query parameter
std::string filename = "";
if (req.has_param("filename")) {
filename = req.get_param_value("filename");
metadata["original_filename"] = filename;
}
if (req.has_param("filename"))
metadata["original_filename"] = params["filename"];
// Check if filename ends with ".tgz" using the utility function
if (utils::ends_with(filename, ".tgz")) {
if (utils::ends_with(params["filename"], ".tgz")) {
metadata["tgz_content_hash"] = get_hash_from_tgz(temp_path.string());
}
add_file_metadata(temp_path.string(), metadata);
// Move file to final location
std::string hash_str = std::to_string(hash);
std::filesystem::path final_path = config_.object_store_path / hash_str;
std::filesystem::path final_path = config_.object_store_path / std::to_string(hash);
if (!std::filesystem::exists(final_path)) {
try {
std::filesystem::rename(temp_path, final_path);
@@ -287,8 +332,8 @@ void Server::handle_put_object(const httplib::Request& req, httplib::Response& r
// Update database index
dbEntry entry;
entry.label_tag = label_tag;
entry.hash = hash_str;
entry.label_tag = params["labeltag"];
entry.hash = std::to_string(hash);
entry.metadata = metadata; // Store the potentially updated metadata
if (!db_->update_or_insert(entry)) {
@@ -299,7 +344,7 @@ void Server::handle_put_object(const httplib::Request& req, httplib::Response& r
return;
}
res.set_content(hash_str, "text/plain");
res.set_content(std::to_string(hash), "text/plain");
}
void Server::handle_get_metadata(const httplib::Request& req, httplib::Response& res) {
@@ -321,10 +366,6 @@ void Server::handle_get_metadata(const httplib::Request& req, httplib::Response&
}
}
bool Server::validate_write_token(const std::string& token) const {
return std::find(config_.write_tokens.begin(), config_.write_tokens.end(), token) != config_.write_tokens.end();
}
std::pair<std::string, std::string> Server::parse_label_tag(const std::string& label_tag) const {
size_t colon_pos = label_tag.find(':');
if (colon_pos == std::string::npos || colon_pos == 0 || colon_pos == label_tag.length() - 1) {
@@ -347,4 +388,109 @@ void Server::add_file_metadata(const std::string &file_path, nlohmann::json &met
metadata["file_modification_time"] = std::chrono::system_clock::to_time_t(sctp);
}
void Server::handle_delete_tag(const httplib::Request& req, httplib::Response& res) {
std::map<std::string, std::string> params;
if (!validate_write_request(req, res, {"token", "labeltag"}, params)) {
return;
}
// Validate label:tag format
auto [label, tag] = parse_label_tag(params["labeltag"]);
if (label.empty() || tag.empty()) {
res.status = 400;
res.set_content("Invalid label:tag format", "text/plain");
return;
}
// Delete the label:tag from the database
if (!db_->remove(params["labeltag"])) {
res.status = 404;
res.set_content("Label:tag not found or deletion failed", "text/plain");
return;
}
res.set_content("Label:tag deleted successfully", "text/plain");
}
void Server::handle_delete_object(const httplib::Request& req, httplib::Response& res) {
std::map<std::string, std::string> params;
if (!validate_write_request(req, res, {"token", "hash"}, params)) {
return;
}
// Validate that the hash exists as a file
std::filesystem::path file_path = config_.object_store_path / params["hash"];
if (!std::filesystem::exists(file_path) || !std::filesystem::is_regular_file(file_path)) {
res.status = 404;
res.set_content("Object not found for hash: " + params["hash"], "text/plain");
return;
}
// Remove all tags that reference this hash
if (!db_->remove_by_hash(params["hash"])) {
res.status = 500;
res.set_content("Failed to remove some or all associated tags", "text/plain");
return;
}
// Remove the file
try {
std::filesystem::remove(file_path);
} catch (const std::filesystem::filesystem_error& e) {
std::cerr << "Error deleting object file: " << e.what() << std::endl;
res.status = 500;
res.set_content("Failed to delete object file: " + std::string(e.what()), "text/plain");
return;
}
res.set_content("Object and all associated tags deleted successfully", "text/plain");
}
void Server::handle_append_tag(const httplib::Request& req, httplib::Response& res) {
std::map<std::string, std::string> params;
if (!validate_write_request(req, res, {"token", "labeltag", "hash"}, params)) {
return;
}
// Validate label:tag format
auto [label, tag] = parse_label_tag(params["labeltag"]);
if (label.empty() || tag.empty()) {
res.status = 400;
res.set_content("Invalid label:tag format", "text/plain");
return;
}
// Validate that the hash exists as a file
std::filesystem::path file_path = config_.object_store_path / params["hash"];
if (!std::filesystem::exists(file_path) || !std::filesystem::is_regular_file(file_path)) {
res.status = 404;
res.set_content("Object not found for hash: " + params["hash"], "text/plain");
return;
}
// Check if the label:tag already exists
dbEntry existing_entry;
if (db_->get(params["labeltag"], existing_entry)) {
if (existing_entry.hash == params["hash"]) {
// Label:tag already points to this hash, nothing to do
res.set_content("Label:tag already points to this hash", "text/plain");
return;
}
}
// Create a new entry with the label:tag pointing to the hash
dbEntry entry;
entry.label_tag = params["labeltag"];
entry.hash = params["hash"];
entry.metadata = nlohmann::json({}); // Empty metadata for appended tags
if (!db_->update_or_insert(entry)) {
res.status = 500;
res.set_content("Failed to append tag", "text/plain");
return;
}
res.set_content("Tag appended successfully", "text/plain");
}
} // namespace simple_object_storage

8
src/server.hpp
View File

@@ -28,12 +28,18 @@ private:
void handle_get_directory(const httplib::Request& req, httplib::Response& res);
void handle_put_object(const httplib::Request& req, httplib::Response& res);
void handle_get_metadata(const httplib::Request& req, httplib::Response& res);
bool validate_write_token(const std::string& token) const;
void handle_delete_tag(const httplib::Request& req, httplib::Response& res);
void handle_delete_object(const httplib::Request& req, httplib::Response& res);
void handle_append_tag(const httplib::Request& req, httplib::Response& res);
std::pair<std::string, std::string> parse_label_tag(const std::string& label_tag) const;
void add_file_metadata(const std::string &file_path, nlohmann::json &metadata) const;
bool init_db();
bool validate_write_request(const httplib::Request& req, httplib::Response& res, const std::vector<std::string>& required_params,
std::map<std::string, std::string>& params);
private:
const ServerConfig& config_;
httplib::Server server_;
std::unique_ptr<Database> db_;

39
test.sh Executable file
View File

@@ -0,0 +1,39 @@
#! /bin/bash
SCRIPT_DIR=$(dirname $0)
SCRIPT_NAME=$(basename $0)
# test jq is installed
if ! command -v jq &> /dev/null; then
echo "jq could not be found"
echo "sudo apt-get install jq"
exit 1
fi
# read ~/.config/simple_object_storage/config.json
CONFIG_PATH="${HOME}/.config/simple_object_storage/config.json"
if [ ! -f "${CONFIG_PATH}" ]; then
echo "config file not found at ${CONFIG_PATH}"
exit 1
fi
CONFIG=$(cat "${CONFIG_PATH}")
# get the host and port from the config
HOST=$(echo $CONFIG | jq -r '.host')
PORT=$(echo $CONFIG | jq -r '.port')
# extract the first write token from the config
WRITE_TOKEN=$(echo $CONFIG | jq -r '.write_tokens[0]')
BASE_URL="http://${HOST}:${PORT}"
BASE_TAG="autotest"
# test every action in the README.md file, leaving the system in the same state it was found
# and print the output of each action
# upload this script as an object
echo "uploading ${SCRIPT_DIR}/${SCRIPT_NAME} to ${BASE_TAG}:test1"
OBJECT_HASH=$(curl "${BASE_URL}/upload?token=${WRITE_TOKEN}&labeltag=${BASE_TAG}:test1&filename=${SCRIPT_NAME}" -T ${SCRIPT_DIR}/${SCRIPT_NAME} | jq -r '.hash')
echo "received hash ${OBJECT_HASH}"