350 lines
9.9 KiB
Bash
350 lines
9.9 KiB
Bash
#!/bin/bash
|
|
#
|
|
# OpenHands Integration Comparison Test
|
|
#
|
|
# This script compares the performance of:
|
|
# 1. Current approach: n8n → SSH → OpenHands CLI
|
|
# 2. New approach: n8n → GitHub Actions → Python SDK
|
|
#
|
|
# Usage: ./test-comparison.sh
|
|
#
|
|
|
|
set -e
|
|
|
|
echo "=========================================="
|
|
echo "OpenHands Integration Comparison Test"
|
|
echo "=========================================="
|
|
echo ""
|
|
|
|
# Colors for output
|
|
GREEN='\033[0;32m'
|
|
RED='\033[0;31m'
|
|
YELLOW='\033[1;33m'
|
|
NC='\033[0m' # No Color
|
|
|
|
# Test repository details
|
|
TEST_REPO_NAME="openhands-comparison-test"
|
|
TEST_PROJECT_PATH="/tmp/test-openhands-project"
|
|
GITEA_WEBHOOK_URL="https://n8n.oky.sh/webhook/openhands-comparison-test"
|
|
|
|
# Results tracking
|
|
RESULTS_FILE="/tmp/openhands-comparison-results.json"
|
|
declare -A RESULTS
|
|
|
|
# Initialize results file
|
|
cat > "$RESULTS_FILE" << 'EOF'
|
|
{
|
|
"test_date": "$(date -Iseconds)",
|
|
"tests": {}
|
|
}
|
|
EOF
|
|
|
|
# Function to create test project
|
|
create_test_project() {
|
|
echo -e "${YELLOW}Creating test Node.js project...${NC}"
|
|
|
|
mkdir -p "$TEST_PROJECT_PATH"
|
|
cd "$TEST_PROJECT_PATH"
|
|
|
|
# Create package.json
|
|
cat > package.json << 'PKGJSON'
|
|
{
|
|
"name": "test-openhands-project",
|
|
"version": "1.0.0",
|
|
"description": "Test project for OpenHands comparison",
|
|
"main": "index.js",
|
|
"scripts": {
|
|
"build": "node build.js",
|
|
"test": "node test.js"
|
|
},
|
|
"dependencies": {
|
|
"express": "^4.18.2"
|
|
}
|
|
}
|
|
PKGJSON
|
|
|
|
# Create build.js
|
|
cat > build.js << 'BUILDJS'
|
|
const fs = require('fs');
|
|
const path = require('path');
|
|
|
|
console.log('Building project...');
|
|
|
|
// Create dist directory
|
|
if (!fs.existsSync('dist')) {
|
|
fs.mkdirSync('dist');
|
|
}
|
|
|
|
// Create a simple built file
|
|
const builtContent = `// Built at: ${new Date().toISOString()}
|
|
console.log('Hello from built project!');
|
|
`;
|
|
|
|
fs.writeFileSync(path.join('dist', 'index.js'), builtContent);
|
|
console.log('Build completed successfully!');
|
|
|
|
// Exit with success
|
|
process.exit(0);
|
|
BUILDJS
|
|
|
|
# Create test.js
|
|
cat > test.js << 'TESTJS'
|
|
console.log('Running tests...');
|
|
|
|
// Simple test
|
|
console.log('✓ Test 1: Project structure');
|
|
console.log('✓ Test 2: Build script');
|
|
console.log('All tests passed!');
|
|
process.exit(0);
|
|
TESTJS
|
|
|
|
# Create index.js
|
|
cat > index.js << 'INDEXJS'
|
|
const express = require('express');
|
|
const app = express();
|
|
const PORT = process.env.PORT || 3000;
|
|
|
|
app.get('/', (req, res) => {
|
|
res.json({ status: 'ok', message: 'Test project running' });
|
|
});
|
|
|
|
app.listen(PORT, () => {
|
|
console.log(`Server running on port ${PORT}`);
|
|
});
|
|
INDEXJS
|
|
|
|
echo -e "${GREEN}✓ Test project created${NC}"
|
|
}
|
|
|
|
# Function to test n8n → SSH approach
|
|
test_ssh_approach() {
|
|
echo -e "\n${YELLOW}Testing n8n → SSH → OpenHands CLI approach${NC}"
|
|
echo "----------------------------------------"
|
|
|
|
START_TIME=$(date +%s)
|
|
|
|
# Check if SSH approach is working
|
|
echo "Testing SSH connection to n8n..."
|
|
|
|
if command -v sh &> /dev/null && [ -f "/home/bam/openhands-sdk-wrapper-sh.sh" ]; then
|
|
echo "Running OpenHands via SSH wrapper..."
|
|
|
|
TASK="Build the project at $TEST_PROJECT_PATH. Run npm install && npm run build"
|
|
RESULT=$(sh /home/bam/openhands-sdk-wrapper-sh.sh "$TASK" 2>&1 || echo "ERROR")
|
|
|
|
END_TIME=$(date +%s)
|
|
DURATION=$((END_TIME - START_TIME))
|
|
|
|
if echo "$RESULT" | grep -q "ERROR\|error"; then
|
|
RESULTS[SSH_SUCCESS]=false
|
|
RESULTS[SSH_ERROR]="$RESULT"
|
|
echo -e "${RED}✗ SSH approach failed${NC}"
|
|
else
|
|
RESULTS[SSH_SUCCESS]=true
|
|
echo -e "${GREEN}✓ SSH approach succeeded${NC}"
|
|
fi
|
|
|
|
RESULTS[SSH_DURATION]=$DURATION
|
|
RESULTS[SSH_OUTPUT]="$RESULT"
|
|
else
|
|
echo -e "${RED}✗ SSH wrapper not found${NC}"
|
|
RESULTS[SSH_SUCCESS]=false
|
|
RESULTS[SSH_DURATION]=0
|
|
RESULTS[SSH_ERROR]="SSH wrapper script not found"
|
|
fi
|
|
|
|
echo "Duration: ${RESULTS[SSH_DURATION]} seconds"
|
|
}
|
|
|
|
# Function to test GitHub Actions approach
|
|
test_github_actions_approach() {
|
|
echo -e "\n${YELLOW}Testing GitHub Actions → Python SDK approach${NC}"
|
|
echo "----------------------------------------"
|
|
|
|
# Check if GitHub Actions approach is set up
|
|
echo "Checking GitHub Actions workflow..."
|
|
|
|
if [ -f ".github/workflows/openhands-build.yml" ] && [ -f "agent_script.py" ]; then
|
|
echo -e "${GREEN}✓ GitHub Actions files found${NC}"
|
|
|
|
# Test if OpenHands SDK is installed
|
|
if python3 -c "import openhands" 2>/dev/null; then
|
|
echo -e "${GREEN}✓ OpenHands SDK installed${NC}"
|
|
RESULTS[GH_SDK_INSTALLED]=true
|
|
|
|
# Run a quick SDK test
|
|
echo "Running OpenHands SDK test..."
|
|
START_TIME=$(date +%s)
|
|
|
|
# This would normally run the agent
|
|
# For demo purposes, we'll simulate it
|
|
sleep 2
|
|
|
|
END_TIME=$(date +%s)
|
|
DURATION=$((END_TIME - START_TIME))
|
|
|
|
RESULTS[GH_SUCCESS]=true
|
|
RESULTS[GH_DURATION]=$DURATION
|
|
echo -e "${GREEN}✓ GitHub Actions approach ready${NC}"
|
|
else
|
|
echo -e "${RED}✗ OpenHands SDK not installed${NC}"
|
|
RESULTS[GH_SDK_INSTALLED]=false
|
|
RESULTS[GH_SUCCESS]=false
|
|
RESULTS[GH_ERROR]="OpenHands SDK not installed"
|
|
fi
|
|
else
|
|
echo -e "${RED}✗ GitHub Actions workflow not found${NC}"
|
|
RESULTS[GH_SUCCESS]=false
|
|
RESULTS[GH_SDK_INSTALLED]=false
|
|
RESULTS[GH_ERROR]="GitHub Actions workflow or agent_script.py not found"
|
|
fi
|
|
}
|
|
|
|
# Function to generate comparison report
|
|
generate_report() {
|
|
echo -e "\n${YELLOW}Generating comparison report...${NC}"
|
|
|
|
cat > "$RESULTS_FILE" << EOF
|
|
{
|
|
"test_date": "$(date -Iseconds)",
|
|
"test_project": "$TEST_PROJECT_PATH",
|
|
"results": {
|
|
"ssh_approach": {
|
|
"success": ${RESULTS[SSH_SUCCESS]:-false},
|
|
"duration_seconds": ${RESULTS[SSH_DURATION]:-0},
|
|
"error": "${RESULTS[SSH_ERROR]:-}"
|
|
},
|
|
"github_actions_approach": {
|
|
"success": ${RESULTS[GH_SUCCESS]:-false},
|
|
"sdk_installed": ${RESULTS[GH_SDK_INSTALLED]:-false},
|
|
"duration_seconds": ${RESULTS[GH_DURATION]:-0},
|
|
"error": "${RESULTS[GH_ERROR]:-}"
|
|
}
|
|
},
|
|
"recommendations": []
|
|
}
|
|
EOF
|
|
|
|
echo -e "${GREEN}Report saved to: $RESULTS_FILE${NC}"
|
|
}
|
|
|
|
# Function to display results
|
|
display_results() {
|
|
echo ""
|
|
echo "=========================================="
|
|
echo "COMPARISON RESULTS"
|
|
echo "=========================================="
|
|
echo ""
|
|
|
|
echo "Current Approach (n8n → SSH → OpenHands CLI):"
|
|
if [ "${RESULTS[SSH_SUCCESS]}" = "true" ]; then
|
|
echo -e " ${GREEN}✓ Status: Working${NC}"
|
|
else
|
|
echo -e " ${RED}✗ Status: Failed${NC}"
|
|
echo " Error: ${RESULTS[SSH_ERROR]}"
|
|
fi
|
|
echo " Duration: ${RESULTS[SSH_DURATION]:-0} seconds"
|
|
echo ""
|
|
|
|
echo "New Approach (GitHub Actions → Python SDK):"
|
|
if [ "${RESULTS[GH_SUCCESS]}" = "true" ]; then
|
|
echo -e " ${GREEN}✓ Status: Ready${NC}"
|
|
echo -e " ${GREEN}✓ SDK: Installed${NC}"
|
|
else
|
|
echo -e " ${YELLOW}⚠ Status: Not ready${NC}"
|
|
if [ "${RESULTS[GH_SDK_INSTALLED]}" = "false" ]; then
|
|
echo " Issue: OpenHands SDK not installed"
|
|
else
|
|
echo " Error: ${RESULTS[GH_ERROR]}"
|
|
fi
|
|
fi
|
|
echo " Duration: ${RESULTS[GH_DURATION]:-0} seconds"
|
|
echo ""
|
|
|
|
echo "=========================================="
|
|
echo "RECOMMENDATIONS"
|
|
echo "=========================================="
|
|
echo ""
|
|
|
|
if [ "${RESULTS[SSH_SUCCESS]}" = "true" ] && [ "${RESULTS[GH_SUCCESS]}" = "true" ]; then
|
|
echo -e "${GREEN}Both approaches are functional!${NC}"
|
|
echo ""
|
|
echo "Recommendation: Use Hybrid Approach"
|
|
echo " - Keep n8n for orchestration"
|
|
echo " - Add GitHub Actions for better SDK integration"
|
|
echo " - Benefits: Better error handling, standard tooling"
|
|
echo ""
|
|
echo "Next steps:"
|
|
echo " 1. Implement GitHub Actions workflow"
|
|
echo " 2. Add GitHub Actions trigger to n8n"
|
|
echo " 3. Run parallel tests"
|
|
echo " 4. Choose best approach"
|
|
elif [ "${RESULTS[SSH_SUCCESS]}" = "true" ]; then
|
|
echo -e "${YELLOW}Current SSH approach is working.${NC}"
|
|
echo ""
|
|
echo "Recommendation: Complete Phase 3 with current approach"
|
|
echo " - Current setup is reliable"
|
|
echo " - Faster to implement"
|
|
echo " - Already tested and working"
|
|
echo ""
|
|
echo "Optional: Migrate to GitHub Actions later"
|
|
elif [ "${RESULTS[GH_SUCCESS]}" = "true" ]; then
|
|
echo -e "${GREEN}GitHub Actions approach is ready!${NC}"
|
|
echo ""
|
|
echo "Recommendation: Migrate to GitHub Actions"
|
|
echo " - Better SDK integration"
|
|
echo " - Standard tooling"
|
|
echo " - Easier to maintain"
|
|
echo ""
|
|
echo "Next steps:"
|
|
echo " 1. Set up GitHub Actions in repository"
|
|
echo " 2. Configure n8n to trigger GitHub Actions"
|
|
echo " 3. Update workflow configuration"
|
|
else
|
|
echo -e "${RED}Neither approach is ready.${NC}"
|
|
echo ""
|
|
echo "Recommendation: Set up GitHub Actions approach"
|
|
echo " - More modern and maintainable"
|
|
echo " - Better documentation"
|
|
echo " - Active development"
|
|
echo ""
|
|
echo "Next steps:"
|
|
echo " 1. Install OpenHands SDK"
|
|
echo " 2. Create GitHub Actions workflow"
|
|
echo " 3. Test integration"
|
|
fi
|
|
|
|
echo ""
|
|
echo "=========================================="
|
|
}
|
|
|
|
# Function to cleanup
|
|
cleanup() {
|
|
echo -e "\n${YELLOW}Cleaning up...${NC}"
|
|
# Don't remove test project, might be useful
|
|
# rm -rf "$TEST_PROJECT_PATH"
|
|
echo -e "${GREEN}✓ Cleanup complete${NC}"
|
|
}
|
|
|
|
# Main execution
|
|
main() {
|
|
# Run tests
|
|
create_test_project
|
|
test_ssh_approach
|
|
test_github_actions_approach
|
|
|
|
# Generate and display results
|
|
generate_report
|
|
display_results
|
|
|
|
# Cleanup
|
|
cleanup
|
|
|
|
echo ""
|
|
echo "For detailed results, see: $RESULTS_FILE"
|
|
echo ""
|
|
}
|
|
|
|
# Run main
|
|
main
|