diff --git a/sos b/sos index 942ccfd..1c20147 100755 --- a/sos +++ b/sos @@ -15,6 +15,12 @@ function die() { } function get_getpkg() { + # Only download getpkg if we don't have sha256sum/shasum + if command -v sha256sum &> /dev/null || command -v shasum &> /dev/null; then + export GETPKG="not_needed" + return + fi + # get getpkg export GETPKG="${SCRIPT_DIR}/../getpkg/output/getpkg" if [ ! -f "${GETPKG}" ]; then @@ -76,7 +82,11 @@ function upload() { # upload the file - TARGET_SERVER="https://$server" + if [ -n "${SOS_TEST_MODE:-}" ]; then + TARGET_SERVER="http://$server" + else + TARGET_SERVER="https://$server" + fi echo "Uploading $file to $TARGET_SERVER" DATETIME=$(datetime) @@ -86,8 +96,15 @@ function upload() { LABELTAGS_JSON=$(printf '"%s",' "${LABELTAGS[@]}") LABELTAGS_JSON="[${LABELTAGS_JSON%,}]" - # trip whitespace from the file path - LOCALHASH=$("${GETPKG}" hash "${file}" | tr -d '[:space:]') + # Calculate SHA-256 hash for deduplication + if command -v sha256sum &> /dev/null; then + LOCALHASH=$(sha256sum "${file}" | cut -d' ' -f1) + elif command -v shasum &> /dev/null; then + LOCALHASH=$(shasum -a 256 "${file}" | cut -d' ' -f1) + else + # Fallback to getpkg hash (note: not SHA-256 compatible) + LOCALHASH=$("${GETPKG}" hash "${file}" | tr -d '[:space:]') + fi echo "Local hash: $LOCALHASH" @@ -110,7 +127,11 @@ EOF fi - EXISTSJSON=$(eval "curl -s \"https://$server/exists/$LOCALHASH\"") || die "Failed to check if file exists" + if [ -n "${SOS_TEST_MODE:-}" ]; then + EXISTSJSON=$(eval "curl -s \"http://$server/exists/$LOCALHASH\"") || die "Failed to check if file exists" + else + EXISTSJSON=$(eval "curl -s \"https://$server/exists/$LOCALHASH\"") || die "Failed to check if file exists" + fi DOESEXIT=$(echo "$EXISTSJSON" | jq -r '.exists') HASH="" @@ -125,27 +146,48 @@ EOF # "$TARGET_SERVER/update" \ # || die "Failed to update metadata at $TARGET_SERVER/update" - curl -X PUT -H "Authorization: Bearer ${WRITE_TOKEN}" \ + UPDATE_RESPONSE=$(curl -s -w "\nHTTP_STATUS:%{http_code}" -X PUT -H "Authorization: Bearer ${WRITE_TOKEN}" \ -F "metadata=${METADATA_JSON}" \ -F "hash=${LOCALHASH}" \ - "$TARGET_SERVER/update" \ - || die "Failed to upload $file to $TARGET_SERVER/update" + "$TARGET_SERVER/update") + + HTTP_STATUS=$(echo "$UPDATE_RESPONSE" | grep "HTTP_STATUS:" | cut -d: -f2) + RESPONSE_BODY=$(echo "$UPDATE_RESPONSE" | sed '/HTTP_STATUS:/d') + + echo "$RESPONSE_BODY" + + if [ "$HTTP_STATUS" != "200" ]; then + die "Failed to update metadata at $TARGET_SERVER/update (HTTP $HTTP_STATUS)" + fi else # UPLOAD the file + metadata - curl -X PUT -H "Authorization: Bearer ${WRITE_TOKEN}" \ + UPLOAD_RESPONSE=$(curl -s -w "\nHTTP_STATUS:%{http_code}" -X PUT -H "Authorization: Bearer ${WRITE_TOKEN}" \ -F "metadata=${METADATA_JSON}" \ -F "file=@${file}" \ - "$TARGET_SERVER/upload" \ - || die "Failed to upload $file to $TARGET_SERVER/upload" + "$TARGET_SERVER/upload") + + HTTP_STATUS=$(echo "$UPLOAD_RESPONSE" | grep "HTTP_STATUS:" | cut -d: -f2) + RESPONSE_BODY=$(echo "$UPLOAD_RESPONSE" | sed '/HTTP_STATUS:/d') + + echo "$RESPONSE_BODY" + + if [ "$HTTP_STATUS" != "200" ]; then + die "Failed to upload $file to $TARGET_SERVER/upload (HTTP $HTTP_STATUS)" + fi fi echo " " echo " " - JSON1=$(eval "curl -s \"https://$server/hash/$first_label\"") - HASH=$(echo "$JSON1" | jq -r '.hash') - - JSON2=$(eval "curl -s \"https://$server/meta/$HASH\"") || die "Failed to get meta for $HASH" + if [ -n "${SOS_TEST_MODE:-}" ]; then + JSON1=$(eval "curl -s \"http://$server/hash/$first_label\"") + HASH=$(echo "$JSON1" | jq -r '.hash') + JSON2=$(eval "curl -s \"http://$server/meta/$HASH\"") || die "Failed to get meta for $HASH" + else + JSON1=$(eval "curl -s \"https://$server/hash/$first_label\"") + HASH=$(echo "$JSON1" | jq -r '.hash') + JSON2=$(eval "curl -s \"https://$server/meta/$HASH\"") || die "Failed to get meta for $HASH" + fi FILENAME=$(echo "$JSON2" | jq -r '.metadata.filename') echo "Metadata:" @@ -153,8 +195,13 @@ EOF echo " " - echo "Download URL: https://$server/$first_label > $FILENAME" - echo "Alternative: https://$server/$HASH > $FILENAME" + if [ -n "${SOS_TEST_MODE:-}" ]; then + echo "Download URL: http://$server/$first_label > $FILENAME" + echo "Alternative: http://$server/$HASH > $FILENAME" + else + echo "Download URL: https://$server/$first_label > $FILENAME" + echo "Alternative: https://$server/$HASH > $FILENAME" + fi } # if no arguments, show help diff --git a/test.sh b/test.sh index 9dbce6c..503bf98 100755 --- a/test.sh +++ b/test.sh @@ -3,18 +3,450 @@ set -euo pipefail SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" +TEST_DIR="${SCRIPT_DIR}/test_tmp" +TEST_TOKEN="test-token-$(date +%s)" +CONTAINER_NAME="sos-test-$(date +%s)" +TEST_PORT=$((7700 + RANDOM % 100)) +CLEANUP_NEEDED=false +# Colors for output +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +NC='\033[0m' # No Color -# spin up a local simple object server to try +function log_info() { + echo -e "${GREEN}[INFO]${NC} $1" +} -# docker run -d \ -# -p 8909:8080 \ -# -v /path/to/storage:/data/storage \ -# -v /path/to/sos_config.json:/data/sos_config.json:ro \ -# --name object-server \ -# gitea.jde.nz/public/simple-object-server +function log_error() { + echo -e "${RED}[ERROR]${NC} $1" >&2 +} +function log_warning() { + echo -e "${YELLOW}[WARN]${NC} $1" +} +function cleanup() { + if [ "$CLEANUP_NEEDED" = true ]; then + log_info "Cleaning up test environment..." + + # Stop and remove container + if docker ps -a --format '{{.Names}}' | grep -q "^${CONTAINER_NAME}$"; then + docker stop "${CONTAINER_NAME}" >/dev/null 2>&1 || true + docker rm "${CONTAINER_NAME}" >/dev/null 2>&1 || true + log_info "Removed test container" + fi + + # Clean up test directory + if [ -d "${TEST_DIR}" ]; then + rm -rf "${TEST_DIR}" + log_info "Removed test directory" + fi + fi +} -"${SCRIPT_DIR}/sos" upload "getbin.xyz" "${SCRIPT_DIR}/sos" "sos:test" "sos:dodgy" +trap cleanup EXIT +function die() { + log_error "$@" + exit 1 +} + +function check_dependencies() { + log_info "Checking dependencies..." + + # Check for Docker + if ! command -v docker &> /dev/null; then + die "Docker is required but not installed" + fi + + # Check for jq + if ! command -v jq &> /dev/null; then + die "jq is required but not installed" + fi + + # Check for curl + if ! command -v curl &> /dev/null; then + die "curl is required but not installed" + fi + + # Check Docker daemon is running + if ! docker info >/dev/null 2>&1; then + die "Docker daemon is not running" + fi + + log_info "All dependencies satisfied" +} + +function setup_test_environment() { + log_info "Setting up test environment..." + CLEANUP_NEEDED=true + + # Create test directory + rm -rf "${TEST_DIR}" + mkdir -p "${TEST_DIR}/storage" + mkdir -p "${TEST_DIR}/test_files" + + # Generate hashed token + log_info "Generating authentication token..." + HASHED_TOKEN=$(docker run --rm gitea.jde.nz/public/simple-object-server /sos/hash_token "${TEST_TOKEN}" 2>/dev/null | grep '^\$2[aby]\$' | head -1) + + if [ -z "${HASHED_TOKEN}" ]; then + die "Failed to generate hashed token" + fi + + # Create test configuration + cat > "${TEST_DIR}/sos_config.json" < "${TEST_DIR}/test_files/test1.txt" + echo "This is test file 2 with more content" > "${TEST_DIR}/test_files/test2.txt" + dd if=/dev/urandom of="${TEST_DIR}/test_files/binary_test.bin" bs=1024 count=10 2>/dev/null + + # Create the sos binary for testing + cp "${SCRIPT_DIR}/sos" "${TEST_DIR}/test_files/sos_binary" + + log_info "Created test files" +} + +function start_test_server() { + log_info "Starting SOS test server on port ${TEST_PORT}..." + + # Start the container (server runs on port 80 inside container) + local container_id=$(docker run -d \ + --name "${CONTAINER_NAME}" \ + -p "${TEST_PORT}:80" \ + -v "${TEST_DIR}/sos_config.json:/data/sos_config.json:ro" \ + -v "${TEST_DIR}/storage:/data/storage" \ + gitea.jde.nz/public/simple-object-server 2>&1) + + if [ $? -ne 0 ]; then + log_error "Failed to start container: ${container_id}" + die "Failed to start test server" + fi + + log_info "Container started with ID: ${container_id:0:12}" + + # Give container a moment to initialize + sleep 2 + + # Check if container is still running + if ! docker ps --format '{{.Names}}' | grep -q "^${CONTAINER_NAME}$"; then + log_error "Container stopped unexpectedly. Logs:" + docker logs "${CONTAINER_NAME}" 2>&1 + die "Container failed to stay running" + fi + + # Wait for server to be ready + log_info "Waiting for server to be ready..." + local max_attempts=30 + local attempt=0 + + while [ $attempt -lt $max_attempts ]; do + # Try multiple endpoints + if curl -s "http://localhost:${TEST_PORT}/health" >/dev/null 2>&1 || \ + curl -s "http://localhost:${TEST_PORT}/" >/dev/null 2>&1; then + log_info "Server is ready!" + return 0 + fi + + # Check container is still running every 5 attempts + if [ $((attempt % 5)) -eq 0 ]; then + if ! docker ps --format '{{.Names}}' | grep -q "^${CONTAINER_NAME}$"; then + log_error "Container stopped while waiting. Logs:" + docker logs "${CONTAINER_NAME}" 2>&1 | tail -20 + die "Container stopped unexpectedly" + fi + fi + + sleep 1 + attempt=$((attempt + 1)) + done + + log_error "Server did not become ready. Container logs:" + docker logs "${CONTAINER_NAME}" 2>&1 | tail -20 + die "Server failed to start after ${max_attempts} seconds" +} + +function test_upload() { + local file="$1" + local label="$2" + shift 2 + local extra_labels="$@" + + log_info "Testing upload: ${file} with label ${label}" + + export SOS_WRITE_TOKEN="${TEST_TOKEN}" + # Override to use HTTP for local testing + export SOS_TEST_MODE=1 + "${SCRIPT_DIR}/sos" upload "localhost:${TEST_PORT}" "${file}" "${label}" ${extra_labels} 2>&1 | tee "${TEST_DIR}/upload_output.txt" + + if [ ${PIPESTATUS[0]} -ne 0 ]; then + log_error "Upload failed for ${file}" + return 1 + fi + + # Verify upload was successful + if grep -q "Download URL:" "${TEST_DIR}/upload_output.txt"; then + log_info "Upload successful for ${file}" + return 0 + else + log_error "Upload verification failed for ${file}" + return 1 + fi +} + +function test_retrieval() { + local identifier="$1" + local expected_content="$2" + + log_info "Testing retrieval by ${identifier}" + + # Try to retrieve the file + local response=$(curl -s "http://localhost:${TEST_PORT}/${identifier}") + + if [ -z "${response}" ]; then + log_error "Failed to retrieve by ${identifier}" + return 1 + fi + + # For hash retrieval, test the actual hash endpoint + if [[ "${identifier}" =~ ^[a-f0-9]{64}$ ]]; then + local meta_response=$(curl -s "http://localhost:${TEST_PORT}/meta/${identifier}") + if echo "${meta_response}" | jq -e '.metadata' >/dev/null 2>&1; then + log_info "Successfully retrieved metadata for hash ${identifier}" + else + log_error "Failed to retrieve metadata for hash ${identifier}" + return 1 + fi + fi + + log_info "Retrieval successful for ${identifier}" + return 0 +} + +function test_duplicate_upload() { + log_info "Testing duplicate file upload (deduplication)..." + + # Create a duplicate file with different name + cp "${TEST_DIR}/test_files/test1.txt" "${TEST_DIR}/test_files/test1_dup.txt" + + export SOS_WRITE_TOKEN="${TEST_TOKEN}" + export SOS_TEST_MODE=1 + + # Upload original + "${SCRIPT_DIR}/sos" upload "localhost:${TEST_PORT}" "${TEST_DIR}/test_files/test1.txt" "dup:original" >/dev/null 2>&1 + + # Upload duplicate (should detect existing file) + local output=$("${SCRIPT_DIR}/sos" upload "localhost:${TEST_PORT}" "${TEST_DIR}/test_files/test1_dup.txt" "dup:copy" 2>&1) + + if echo "${output}" | grep -q "File already exists, skipping upload"; then + log_info "Deduplication working correctly" + return 0 + else + log_error "Deduplication test failed" + return 1 + fi +} + +function test_metadata_update() { + log_info "Testing metadata update..." + + export SOS_WRITE_TOKEN="${TEST_TOKEN}" + export SOS_TEST_MODE=1 + + # Upload a file + "${SCRIPT_DIR}/sos" upload "localhost:${TEST_PORT}" "${TEST_DIR}/test_files/test2.txt" "meta:v1" >/dev/null 2>&1 + + # Upload same file with different metadata + local output=$("${SCRIPT_DIR}/sos" upload "localhost:${TEST_PORT}" "${TEST_DIR}/test_files/test2.txt" "meta:v2" "meta:updated" 2>&1) + + if echo "${output}" | grep -q "File already exists"; then + log_info "Metadata update successful" + return 0 + else + log_error "Metadata update test failed" + return 1 + fi +} + +function test_exists_endpoint() { + log_info "Testing exists endpoint..." + + # Get hash of an uploaded file + local hash=$(curl -s "http://localhost:${TEST_PORT}/hash/test:file1" | jq -r '.hash') + + if [ -z "${hash}" ] || [ "${hash}" = "null" ]; then + log_error "Failed to get hash for exists test" + return 1 + fi + + # Test exists endpoint + local exists=$(curl -s "http://localhost:${TEST_PORT}/exists/${hash}" | jq -r '.exists') + + if [ "${exists}" = "true" ]; then + log_info "Exists endpoint working correctly" + return 0 + else + log_error "Exists endpoint test failed" + return 1 + fi +} + +function test_invalid_auth() { + log_info "Testing invalid authentication..." + + # Create a unique test file for this test + echo "auth test content" > "${TEST_DIR}/test_files/auth_test.txt" + + export SOS_WRITE_TOKEN="invalid-token" + export SOS_TEST_MODE=1 + + # Run upload with invalid token and capture output + exit code + set +e # Temporarily disable exit on error + "${SCRIPT_DIR}/sos" upload "localhost:${TEST_PORT}" "${TEST_DIR}/test_files/auth_test.txt" "auth:test" 2>&1 | tee "${TEST_DIR}/auth_test_output.txt" + local exit_code=${PIPESTATUS[0]} + set -e # Re-enable exit on error + + local output=$(cat "${TEST_DIR}/auth_test_output.txt") + + # Check if it failed with non-zero exit code and error message + if [ ${exit_code} -ne 0 ] && (echo "${output}" | grep -q "Failed to upload" || echo "${output}" | grep -q "Invalid write token"); then + log_info "Invalid auth correctly rejected" + return 0 + else + log_error "Invalid auth test failed - upload should have been rejected (exit code: ${exit_code})" + echo "Output was: ${output}" + return 1 + fi +} + +function run_tests() { + local total_tests=0 + local passed_tests=0 + + # Disable exit on error for test functions + set +e + + log_info "Starting test suite..." + echo "" + + # Test 1: Basic upload + ((total_tests++)) + test_upload "${TEST_DIR}/test_files/test1.txt" "test:file1" + if [ $? -eq 0 ]; then + ((passed_tests++)) + fi + echo "" + + # Test 2: Upload with multiple labels + ((total_tests++)) + test_upload "${TEST_DIR}/test_files/test2.txt" "test:file2" "version:1.0" "env:test" + if [ $? -eq 0 ]; then + ((passed_tests++)) + fi + echo "" + + # Test 3: Binary file upload + ((total_tests++)) + test_upload "${TEST_DIR}/test_files/binary_test.bin" "binary:test" + if [ $? -eq 0 ]; then + ((passed_tests++)) + fi + echo "" + + # Test 4: Upload sos binary itself + ((total_tests++)) + test_upload "${TEST_DIR}/test_files/sos_binary" "sos:test" "sos:latest" + if [ $? -eq 0 ]; then + ((passed_tests++)) + fi + echo "" + + # Test 5: Retrieval by label + ((total_tests++)) + test_retrieval "test:file1" "This is test file 1" + if [ $? -eq 0 ]; then + ((passed_tests++)) + fi + echo "" + + # Test 6: Deduplication + ((total_tests++)) + test_duplicate_upload + if [ $? -eq 0 ]; then + ((passed_tests++)) + fi + echo "" + + # Test 7: Metadata update + ((total_tests++)) + test_metadata_update + if [ $? -eq 0 ]; then + ((passed_tests++)) + fi + echo "" + + # Test 8: Exists endpoint + ((total_tests++)) + test_exists_endpoint + if [ $? -eq 0 ]; then + ((passed_tests++)) + fi + echo "" + + # Test 9: Invalid authentication + ((total_tests++)) + test_invalid_auth + if [ $? -eq 0 ]; then + ((passed_tests++)) + fi + echo "" + + # Re-enable exit on error + set -e + + # Summary + echo "==================================" + if [ ${passed_tests} -eq ${total_tests} ]; then + echo -e "${GREEN}All tests passed!${NC} (${passed_tests}/${total_tests})" + return 0 + else + echo -e "${RED}Some tests failed!${NC} (${passed_tests}/${total_tests})" + return 1 + fi +} + +function main() { + echo "SOS Test Suite" + echo "==============" + echo "" + + check_dependencies + setup_test_environment + start_test_server + + echo "" + run_tests + local test_result=$? + + echo "" + log_info "Test suite completed" + + # Show container logs if tests failed + if [ ${test_result} -ne 0 ]; then + log_warning "Showing server logs for debugging:" + docker logs "${CONTAINER_NAME}" 2>&1 | tail -20 + fi + + exit ${test_result} +} + +main "$@" \ No newline at end of file