Tests and POST functionality.

Author: benubois

.github/workflows/ci.yml

@@ -3,21 +3,29 @@ name: CI
 on: [push, pull_request]
 
 jobs:
-  build:
-
-    runs-on: ubuntu-latest
-
+  test:
     strategy:
+      fail-fast: false
       matrix:
-        node-version: [18.x]
+        os: [ubuntu-24.04]
+        ruby: [3.4.4]
+
+    runs-on: ${{ matrix.os }}
 
     steps:
-      - uses: actions/checkout@v3
-      - name: Use Node.js ${{ matrix.node-version }}
-        uses: actions/setup-node@v3
-        with:
-          node-version: ${{ matrix.node-version }}
-          cache: 'npm'
-      - run: npm install
-      - run: npm run build --if-present
-      - run: npm test
+    - uses: actions/checkout@v2
+
+    - uses: actions/setup-node@v4
+      with:
+        node-version: 20
+        cache: "npm"
+    - run: npm ci
+
+    - name: Set up Ruby
+      uses: ruby/setup-ruby@v1
+      with:
+        ruby-version: ${{ matrix.ruby }}
+        bundler-cache: true
+
+    - name: Run tests
+      run: bundle exec rake

Gemfile

@@ -6,4 +6,10 @@ gem "sd_notify"
 gem "unicorn"
 gem "sinatra"
 gem "connection_pool"
-gem "http"
+gem "http"
+
+group :test do
+  gem "minitest"
+  gem "webmock"
+  gem "rack-test"
+end

Gemfile.lock

@@ -4,12 +4,19 @@ GEM
     addressable (2.8.7)
       public_suffix (>= 2.0.2, < 7.0)
     base64 (0.3.0)
+    bigdecimal (3.2.2)
     connection_pool (2.5.3)
+    crack (1.0.0)
+      bigdecimal
+      rexml
     domain_name (0.6.20240107)
     ffi (1.17.2)
+    ffi (1.17.2-arm64-darwin)
+    ffi (1.17.2-x86_64-linux-gnu)
     ffi-compiler (1.3.2)
       ffi (>= 1.15.5)
       rake
+    hashdiff (1.2.0)
     http (5.3.1)
       addressable (~> 2.8)
       http-cookie (~> 1.0)
@@ -23,6 +30,7 @@ GEM
       ffi-compiler (~> 1.0)
       rake (~> 13.0)
     logger (1.7.0)
+    minitest (5.25.5)
     mustermann (3.0.3)
       ruby2_keywords (~> 0.0.1)
     nio4r (2.7.4)
@@ -37,8 +45,11 @@ GEM
     rack-session (2.1.1)
       base64 (>= 0.1.0)
       rack (>= 3.0.0)
+    rack-test (2.2.0)
+      rack (>= 1.3)
     raindrops (0.20.1)
     rake (13.3.0)
+    rexml (3.4.1)
     ruby2_keywords (0.0.5)
     sd_notify (0.1.1)
     sinatra (4.1.1)
@@ -52,17 +63,25 @@ GEM
     unicorn (6.1.0)
       kgio (~> 2.6)
       raindrops (~> 0.7)
+    webmock (3.25.1)
+      addressable (>= 2.8.0)
+      crack (>= 0.3.2)
+      hashdiff (>= 0.4.0, < 2.0.0)
 
 PLATFORMS
-  ruby
+  arm64-darwin-24
+  x86_64-linux
 
 DEPENDENCIES
   connection_pool
   http
+  minitest
   puma
+  rack-test
   sd_notify
   sinatra
   unicorn
+  webmock
 
 BUNDLED WITH
    2.6.7

README.md

@@ -61,7 +61,7 @@ cd extract
 mkdir users
 
 # use your own secret key and username
-echo "SECRET_KEY" > users/USERNAME
+echo "user: key" > users/users.yml
 ```
 
 Once a username and password has been created, you can make a request.

Rakefile

@@ -0,0 +1,9 @@
+require "rake/testtask"
+
+Rake::TestTask.new(:test) do |t|
+  t.libs << "_test"
+  t.test_files = FileList["test/**/*_test.rb"]
+  t.warning = false
+end
+
+task default: :test

app/app.rb

@@ -5,24 +5,40 @@
 require "openssl"
 require "base64"
 require "connection_pool"
+require "yaml"
 
 set :protection, except: [:json_csrf]
 
 $parser = ConnectionPool.new(size: 1, timeout: 5) {
   HTTP.persistent(ENV["PARSER_URL"])
 }
 
+$users = begin
+  if ENV["EXTRACT_USERS"]
+    YAML.safe_load_file(ENV["EXTRACT_USERS"])
+  else
+    {"demo" => "demo"}
+  end
+end
+
+
 def signature_valid?(user, signature, data)
-  path = File.expand_path(File.join("..", "users", user), __dir__)
-  key = File.read(path).strip
+  key = $users[user]
+  return false unless key
+
   signature == OpenSSL::HMAC.hexdigest("sha1", key, data)
 end
 
-def parse(json)
+def parse_with_mercury(json)
   $parser.with do |connection|
-    connection
+    response = connection
       .timeout(connect: 1, write: 5, read: 5)
       .post("/parser", json: json)
+
+    body = response.to_s
+    halt_with_error("Cannot extract this URL.") unless response.status.ok?
+    headers("Content-Type" => response.headers[:content_type])
+    body
   end
 end
 
@@ -33,20 +49,37 @@ def halt_with_error(error)
   }.to_json
 end
 
+def parser_object(url:, html:, content_type:)
+  {
+    url: url,
+    options: {
+      html: html,
+      contentType: content_type
+    }
+  }
+end
+
 def download_with_http(url)
   response = HTTP
     .follow(max_hops: 5)
     .timeout(connect: 4, write: 4, read: 5)
     .headers({accept_encoding: "gzip, deflate"})
     .use(:auto_inflate)
     .get(url)
-  {
-    url: url,
-    options: {
-      html: response.to_s,
-      contentType: response.headers[:content_type]
-    }
-  }
+
+  parser_object(url: url, html: response.to_s, content_type: response.headers[:content_type])
+end
+
+def authenticate(user, signature, url)
+  halt_with_error("User does not exist: #{user}.") unless $users.key?(user)
+  halt_with_error("Invalid signature.") unless signature_valid?(user, signature, url)
+end
+
+def response_error!(exception, url, user)
+  logger.error "Exception processing exception=#{exception} url=#{url} user=#{user} "
+  logger.error exception.backtrace.join("\n")
+  halt_with_error("Cannot extract this URL.")
+  raise exception
 end
 
 get "/health_check" do
@@ -62,21 +95,32 @@ def download_with_http(url)
 
   logger.info "url=#{url}"
 
-  begin
-    halt_with_error("Invalid signature.") unless signature_valid?(params["user"], params["signature"], url)
-  rescue Errno::ENOENT
-    halt_with_error("User does not exist: #{params["user"]}.")
-  end
+  authenticate(params["user"], params["signature"], url)
 
   payload = download_with_http(url)
-  response = parse(payload)
-  body = response.to_s
-  halt_with_error("Cannot extract this URL.") unless response.status.ok?
-  headers("Content-Type" => response.headers[:content_type])
-  body
+
+  parse_with_mercury(payload)
 rescue => exception
-  logger.error "Exception processing exception=#{exception} url=#{url} user=#{params["user"]} "
-  logger.error exception.backtrace.join("\n")
-  halt_with_error("Cannot extract this URL.")
-  raise exception
+  response_error!(exception, url, params["user"])
+end
+
+post "/parser/:user/:signature" do
+  json = begin
+    JSON.parse(request.body.read)
+  rescue JSON::ParserError
+    halt_with_error("Invalid JSON body.")
+  end
+
+  halt_with_error("Missing url field in JSON body.") unless json["url"]
+  halt_with_error("Missing body field in JSON body.") unless json["body"]
+
+  logger.info "url=#{json["url"]}"
+
+  authenticate(params["user"], params["signature"], json["url"])
+
+  payload = parser_object(url: json["url"], html: json["body"], content_type: "text/html")
+
+  parse_with_mercury(payload)
+rescue => exception
+  response_error!(exception, url, params["user"])
 end

script/bootstrap.sh

@@ -0,0 +1,55 @@
+require "webmock/minitest"
+require_relative "test_helper"
+
+class AppTest < Test
+
+  def test_health_check
+    get "/health_check"
+    assert_equal 200, last_response.status
+    assert_equal "OK", last_response.body
+  end
+
+  def test_parser_with_valid_signature
+    url = "https://example.com"
+    base64_url = Base64.urlsafe_encode64(url)
+    signature = OpenSSL::HMAC.hexdigest("sha1", @key, url)
+    title = "The Title"
+
+    stub_request(:get, url)
+      .to_return(
+        status: 200,
+        body: "<title>#{title}</title>",
+        headers: {"Content-Type" => "text/html"}
+      )
+
+    get "/parser/#{@user}/#{signature}?base64_url=#{base64_url}"
+
+    assert_equal 200, last_response.status
+    assert_equal "application/json; charset=utf-8", last_response.content_type
+    assert_equal title, JSON.load(last_response.body).fetch("title")
+  end
+
+  def test_parser_with_invalid_signature
+    url = "https://example.com"
+    base64_url = Base64.urlsafe_encode64(url)
+    invalid_signature = "invalid"
+
+    get "/parser/#{@user}/#{invalid_signature}?base64_url=#{base64_url}"
+
+    assert_equal 400, last_response.status
+    assert_equal "application/json", last_response.content_type
+    assert_equal "Invalid signature.", JSON.parse(last_response.body).fetch("messages")
+  end
+
+  def test_post_parser_with_valid_signature
+    url = "https://example.com"
+    signature = OpenSSL::HMAC.hexdigest("sha1", @key, url)
+    title = "The Title"
+    html_body = "<title>#{title}</title>"
+
+    post "/parser/#{@user}/#{signature}", {url: url, body: html_body}.to_json, "CONTENT_TYPE" => "application/json"
+
+    assert_equal "application/json; charset=utf-8", last_response.content_type
+    assert_equal title, JSON.load(last_response.body).fetch("title")
+  end
+end