Merge https://github.com/toddmotto/public-apis

# Conflicts:
#	README.md

.travis.yml

-language: ruby
-rvm: 2.2
-before_script: gem install awesome_bot
-script: awesome_bot README.md --allow-ssl --allow 403,302
+language: node_js
 notifications:
   email: false
+before_install:
+  - rvm install 2.4.0
+install:
+  - gem install httparty ruby-progressbar
+before_script:
+  - cd build
+script:
+  - ./main.sh
+after_success:
+  - ./build.sh
+  - ./deploy.sh

CONTRIBUTING.md

@@ -31,6 +31,8 @@ Please continue to follow the alphabetical ordering that is in place per section
 
 If an API seems to fall into multiple categories, please place the listing within the section most in line with the services offered through the API. For example, the Instagram API is listed under `Social` since it is mainly a social network, even though it could also apply to `Photography`.
 
+This project now offers a [JSON version](json/) of the list of services. These files are automatically updated as part of the Continuous Integration process, so there is no need to update them as part of the contribution. 
+
 ## Pull Request
 
 After you've created a branch on your fork with your changes, it's time to [make a pull request][pr-link]!

build/build.sh

+#!/bin/bash
+
+# create json directory if not already present
+mkdir -p ../json
+# parse API README and print (minified) JSON to stdout, redirect to /json
+node md2json.js ../README.md > ../json/entries.min.json
+# beautify the previously created JSON file, redirect to /json
+python -m json.tool ../json/entries.min.json > ../json/entries.json

build/deploy.sh

+#!/bin/bash
+
+set -o errexit -o nounset
+
+if [ "$TRAVIS_BRANCH" != "master" ]
+then
+  echo "This commit was made against $TRAVIS_BRANCH and not master! No deploy!"
+  exit 0
+fi
+
+rev=$(git rev-parse --short HEAD)
+
+mkdir deploy
+cd deploy
+
+git init
+git config --global user.name $GH_USER
+git config --global user.email $GH_EMAIL
+
+git remote add upstream "https://$GH_TOKEN@github.com/toddmotto/public-apis.git"
+git fetch upstream
+git reset upstream/master
+
+mv ../../json .
+
+git add json/
+git commit -m "rebuild JSON at ${rev}" -m "[ci skip]"
+git push upstream HEAD:master
+

build/main.sh

+#!/bin/bash
+
+FORMAT_FILE=../README.md
+if [ "$TRAVIS_PULL_REQUEST" == "false" ]; then
+    echo "running on $TRAVIS_BRANCH branch"
+    LINK_FILE=../README.md
+else
+    echo "running on Pull Request #$TRAVIS_PULL_REQUEST"
+    DIFF_URL="https://patch-diff.githubusercontent.com/raw/toddmotto/public-apis/pull/$TRAVIS_PULL_REQUEST.diff"
+    curl $DIFF_URL > diff.txt
+    echo "------- BEGIN DIFF -------"
+    cat diff.txt
+    echo "-------- END DIFF --------"
+    cat diff.txt | egrep "\+" > additions.txt
+    echo "------ BEGIN ADDITIONS -----"
+    cat additions.txt
+    echo "------- END ADDITIONS ------"
+    LINK_FILE=additions.txt
+fi
+
+echo "running format validation..."
+./validate_format.rb $FORMAT_FILE
+if [[ $? != 0 ]]; then
+    echo "format validation failed!"
+    exit 1
+else
+    echo "format validation passed!"
+fi
+
+echo "running link validation..."
+./validate_links.rb $LINK_FILE
+if [[ $? != 0 ]]; then
+    echo "link validation failed!"
+    exit 1
+else
+    echo "link validation passed!"
+fi

build/md2json.js

+fs = require('fs')
+
+function md_trim(str, context) {
+    str = str.replace(/(^\s+)|(\s+$)/g, "");
+
+    if (context == 1) { // Name
+        // placeholder for any formatting on name value
+    } else if (context == 2) { // Description
+        str = str.replace(".", ""); // remove ending periods on descriptions
+    } else if (context == 3) { // Auth
+        if (str.toUpperCase() == "NO") {
+            str = null
+        } else {
+            str = str.replace("`", "").replace("`", "")
+        }
+    } else if (context == 4) { // HTTPS
+        if (str.toUpperCase() == "YES") {
+            str = true
+        } else {
+            str = false
+        }
+    } else if (context == 5) { // Link
+        str = str.replace("[Go!]", "").slice(1, -1);
+    }
+    return str;
+}
+
+function handle(filename, anchor) {
+    fs.readFile(filename, 'utf8', function (err,text) {
+      if (err) {
+        return console.log(err);
+      }
+    var lines = text.split("\n");
+    var cur_line = 0;
+    var line = ""
+    var table_name = "";
+    var col_num = 0;
+    var cols = [];
+    var rows = [];
+
+    function read_line() {
+        return lines[cur_line++];
+    }
+    var root = {};
+    while (true) {
+        var cols = [];
+        var rows = [];
+        while (line.indexOf(anchor) == -1 && cur_line != lines.length) {
+            line = read_line();
+        }
+        if (cur_line == lines.length) {
+            break;
+        }
+        table_name = line.split(anchor)[1];
+        table_name = md_trim(table_name, 0)
+
+        line = read_line()
+
+        if (line) {
+            line = line.split("|")
+            for (var j in line) {
+
+                line[j] = md_trim(line[j], 0)
+                if ((j == 0 || j == line.length - 1) && line[j] === "") {
+
+                } else {
+                    cols.push(line[j]);
+                }
+            }
+            if (line.length) {
+                cols = line;
+                rows.push(cols)
+            } else {
+                console.error("markdown expect column title")
+                break;
+            }
+        } else {
+            console.error("markdown expect table content")
+            break;
+        }
+
+        line = read_line()
+
+        if (!line) {
+            console.error("markdown expect table spliter")
+            break;
+        }
+        line = read_line()
+        while (line.indexOf("|") != -1 && cur_line != lines.length) {
+
+            var line_this = line.split("|")
+            var row = []
+            for (var j in line_this) {
+                line_this[j] = md_trim(line_this[j], j)
+                if ((j == 0 || j == line_this.length - 1) && line_this[j] === "") {
+
+                } else {
+                    row.push(line_this[j]);
+                }
+
+            }
+            rows.push(row);
+            line = read_line()
+        }
+
+        var data=[];
+        for (var j in rows) {
+            if (j != 0) {
+                var ele = {};
+                for (var k in rows[j]) {
+                    ele[rows[0][k]] = rows[j][k];
+                }
+                data.push(ele);
+            }
+        }
+        root[table_name] = data;
+    }
+    console.log(JSON.stringify(root));
+  });
+}
+
+if (process.argv.length < 3) {
+    console.log("No .md file passed!");
+    return;
+}
+if (process.argv.length < 4) {
+  anchorText = "###";
+} else {
+  anchorText = process.argv[3];
+}
+handle(process.argv[2].toString(), anchorText);

build/validate_format.rb

+#!/usr/bin/env ruby
+auth_keys = ['apiKey', 'OAuth', 'X-Mashape-Key', 'No']
+https_keys = ['Yes', 'No']
+args = ARGV
+filename = args[0]
+fail_flag = false
+File.foreach(filename).with_index do |line, line_num|
+    line_num += 1
+    if line.start_with?('|')
+        # Skip table schema lines
+        if line.eql? "|---|---|---|---|---|\n"
+            next
+        end
+        values = line.split("|")
+        # Check Description to make sure first character is capitalized
+        desc_val = values[2].lstrip.chop
+        if !/[[:upper:]]/.match(desc_val[0])
+            puts "(#{line_num}) Invalid Description (first char not uppercase): #{desc_val}"
+            fail_flag = true
+        end
+        # Check Auth values to conform to valid options only
+        auth_val = values[3].lstrip.chop.tr('``', '')
+        if !auth_keys.include?(auth_val)
+            puts "(#{line_num}) Invalid Auth (not a valid option): #{auth_val}"
+            fail_flag = true
+        end
+        # Check HTTPS Support values to be either "Yes" or "No"
+        https_val = values[4].lstrip.chop
+        if !https_keys.include?(https_val)
+            puts "(#{line_num}) Invalid HTTPS: (must use \"Yes\" or \"No\"): #{https_val}"
+            fail_flag = true
+        end
+        # Check Link to ensure url is wrapped in "[Go!]" view
+        link_val = values[5].lstrip.chop
+        if !link_val.start_with?("[Go!](") || !link_val.end_with?(')')
+            puts "(#{line_num}) Invalid Link: (format should be \"[Go!](<LINK>)\"): #{link_val}"
+            fail_flag = true
+        end
+    end
+end
+if fail_flag
+    exit(1)
+else
+    exit(0)
+end

build/validate_links.rb

+#!/usr/bin/env ruby
+require 'httparty'
+require 'ruby-progressbar'
+require 'uri'
+allowed_codes = [200, 302, 403, 429]
+allowed_links = ["https://www.yelp.com/developers/documentation/v3"]
+args = ARGV
+filename = args[0]
+contents = File.open(filename, 'rb') { |f| f.read }
+raw_links = URI.extract(contents, ['http', 'https'])
+# Remove trailing ')' from entry URLs
+links = []
+raw_links.each do |link|
+    if link.end_with?(')')
+        links.push(link[0...-1])
+    else
+        links.push(link)
+    end
+end
+if links.length <= 0
+    puts "no links to check"
+    exit(0)
+end
+fails = []
+# Fail on any duplicate elements
+dup = links.select{|element| links.count(element) > 1}
+if dup.uniq.length > 0
+    dup.uniq.each do |e|
+        fails.push("(DUP): #{e}")
+    end
+end
+# Remove any duplicates from array
+links = links.uniq
+count = 0
+total = links.length
+progressbar = ProgressBar.create(:total => total,
+    :format => "%a %P% | Processed: %c from %C")
+# GET each link and check for valid response code from allowed_codes
+links.each do |link|
+    begin
+        count += 1
+        if allowed_links.include?(link)
+            next
+        end
+        res = HTTParty.get(link, timeout: 10)
+        if res.code.nil?
+            fails.push("(NIL): #{link}")
+            next
+        end
+        if !allowed_codes.include?(res.code)
+            fails.push("(#{res.code}): #{link}")
+        end
+    rescue Net::ReadTimeout
+        fails.push("(TMO): #{link}")
+    rescue Net::OpenTimeout
+        fails.push("(TMO): #{link}")
+    rescue OpenSSL::SSL::SSLError
+        fails.push("(SSL): #{link}")
+    rescue SocketError
+        fails.push("(SOK): #{link}")
+    rescue Errno::ECONNREFUSED
+        fails.push("(CON): #{link}")
+    end
+    progressbar.increment
+end
+puts "#{count}/#{total} links checked"
+if fails.length <= 0
+    puts "all links valid"
+    exit(0)
+else
+    puts "-- RESULTS --"
+    fails.sort!
+    fails.each do |e|
+        puts e
+    end
+    exit(1)
+end