Merge pull request #386 from davemachado/hotfix/update-link-validation-engine

Update Link Validation

.travis.yml

@@ -4,7 +4,7 @@ notifications:
 before_install:
   - rvm install 2.4.0
 install:
-  - gem install awesome_bot
+  - gem install httparty ruby-progressbar
 before_script:
   - cd build
 script:
@@ -12,4 +12,3 @@ script:
 after_success:
   - ./build.sh
   - ./deploy.sh
-

build/main.sh

@@ -1,16 +1,20 @@
 #!/bin/bash
-
 echo "running format validation..."
-./validate.rb ../README.md
+./validate_format.rb ../README.md
-
 if [[ $? != 0 ]]; then
     echo "format validation failed!"
-    exit $?
+    exit 1
 else
     echo "format validation passed!"
 fi
 
 if [ "$TRAVIS_BRANCH" == "master" ]; then
     echo "running link validation..."
-    awesome_bot ../README.md --allow-ssl --allow 403,302
+    ./validate_links.rb ../README.md
+    if [[ $? != 0 ]]; then
+        echo "link validation failed!"
+        exit 1
+    else
+        echo "link validation passed!"
+    fi
 fi

build/validate_format.rb

@@ -1,11 +1,9 @@
 #!/usr/bin/env ruby
 auth_keys = ['apiKey', 'OAuth', 'X-Mashape-Key', 'No']
 https_keys = ['Yes', 'No']
-
 args = ARGV
 filename = args[0]
 fail_flag = false
-
 File.foreach(filename).with_index do |line, line_num|
     line_num += 1
     if line.start_with?('|')
@@ -14,28 +12,24 @@ File.foreach(filename).with_index do |line, line_num|
             next
         end
         values = line.split("|")
-
         # Check Description to make sure first character is capitalized
         desc_val = values[2].lstrip.chop
         if !/[[:upper:]]/.match(desc_val[0])
             puts "(#{line_num}) Invalid Description (first char not uppercase): #{desc_val}"
             fail_flag = true
         end
-
         # Check Auth values to conform to valid options only
         auth_val = values[3].lstrip.chop.tr('``', '')
         if !auth_keys.include?(auth_val)
             puts "(#{line_num}) Invalid Auth (not a valid option): #{auth_val}"
             fail_flag = true
         end
-
         # Check HTTPS Support values to be either "Yes" or "No"
         https_val = values[4].lstrip.chop
         if !https_keys.include?(https_val)
             puts "(#{line_num}) Invalid HTTPS: (must use \"Yes\" or \"No\"): #{https_val}"
             fail_flag = true
         end
-
         # Check Link to ensure url is wrapped in "[Go!]" view
         link_val = values[5].lstrip.chop
         if !link_val.start_with?("[Go!](") || !link_val.end_with?(')')
@@ -44,7 +38,6 @@ File.foreach(filename).with_index do |line, line_num|
         end
     end
 end
-
 if fail_flag
     exit(1)
 else

build/validate_links.rb

@@ -0,0 +1,66 @@
+#!/usr/bin/env ruby
+require 'httparty'
+require 'ruby-progressbar'
+require 'uri'
+allowed_codes = [200, 302, 403]
+args = ARGV
+filename = args[0]
+contents = File.open(filename, 'rb') { |f| f.read }
+raw_links = URI.extract(contents, ['http', 'https'])
+# Remove trailing ')' from entry URLs
+links = []
+raw_links.each do |link|
+    if link.end_with?(')')
+        links.push(link[0...-1])
+    else
+        links.push(link)
+    end
+end
+fails = []
+# Fail on any duplicate elements
+dup = links.select{|element| links.count(element) > 1}
+if dup.uniq.length > 0
+    dup.uniq.each do |e|
+        fails.push("(DUP): #{e}")
+    end
+end
+# Remove any duplicates from array
+links = links.uniq
+count = 0
+total = links.length
+progressbar = ProgressBar.create(:total => total)
+# GET each link and check for valid response code from allowed_codes
+links.each do |link|
+    begin
+        count += 1
+        res = HTTParty.get(link, timeout: 10)
+        if res.code.nil?
+            fails.push("(NIL): #{link}")
+            next
+        end
+        if !allowed_codes.include?(res.code)
+            fails.push("(#{res.code}): #{link}")
+        end
+    rescue Net::ReadTimeout
+        fails.push("(TMO): #{link}")
+    rescue OpenSSL::SSL::SSLError
+        fails.push("(SSL): #{link}")
+    rescue SocketError
+        fails.push("(SOK): #{link}")
+    rescue Errno::ECONNREFUSED
+        fails.push("(CON): #{link}")
+    end
+    progressbar.increment
+end
+puts "#{count}/#{total} links checked"
+if fails.length <= 0
+    puts "all links valid"
+    exit(0)
+else
+    puts "-- RESULTS --"
+    fails.sort!
+    fails.each do |e|
+        puts e
+    end
+    exit(1)
+end