Merge pull request #1 from davemachado/feature/markdown-tables-to-json

Feature/markdown tables to json

.travis.yml

-language: ruby
-rvm: 2.2
-before_script: gem install awesome_bot
-script: awesome_bot README.md --allow-ssl --allow 403
+language: node_js
+before_install:
+  - rvm install 2.4.0
+install:
+  - gem install awesome_bot
+script:
+  - awesome_bot README.md --allow-ssl --allow 403
+after_script:
+  - cd build
+  - sh build.sh
+  - sh deploy.sh
 notifications:
   email: false

README.md

@@ -48,24 +48,21 @@ Please note a passing build status indicates all listed APIs are available since
 * [Weather](#weather)
 
 ### Animals
-
-| API | Description | Auth | HTTPS | Link |
+API | Description | Auth | HTTPS | Link |
 |---|---|---|---|---|
 | IUCN | IUCN Red List of Threatened Species | `token` | No | [Go!](http://apiv3.iucnredlist.org/api/v3/docs) |
 | Petfinder | Adoption | Yes | Yes | [Go!](https://www.petfinder.com/developers/api-docs/) |
 | RescueGroups | Adoption | No | Yes | [Go!](https://userguide.rescuegroups.org/display/APIDG/API+Developers+Guide+Home) |
 
 ### Anime
-
-| API | Description | Auth | HTTPS | Link |
+API | Description | Auth | HTTPS | Link |
 |---|---|---|---|---|
 | AniList | AniList Anime | `OAuth` | No | [Go!](http://anilist-api.readthedocs.io/en/latest/) |
 | Kitsu | Kitsu Anime | `OAuth` | No | [Go!](http://docs.kitsu17.apiary.io/) |
 | Studio Ghibli | Resources from Studio Ghibli films | No | Yes | [Go!](https://ghibliapi.herokuapp.com) |
 
 ### Anti-Malware
-
-| API | Description | Auth | HTTPS | Link |
+API | Description | Auth | HTTPS | Link |
 |---|---|---|---|---|
 | Certly | Certly Link/Domain Flagging | `token` | Yes | [Go!](https://guard.certly.io/) |
 | Google Safe Browsing | Google Link/Domain Flagging | `token` | Yes | [Go!](https://developers.google.com/safe-browsing/) |
@@ -74,8 +71,7 @@ Please note a passing build status indicates all listed APIs are available since
 | Web Of Trust (WOT) | Website reputation | `apiKey` | Yes | [Go!](https://www.mywot.com/wiki/API) |
 
 ### Art & Design
-
-| API | Description | Auth | HTTPS | Link |
+API | Description | Auth | HTTPS | Link |
 |---|---|---|---|---|
 | Dribbble | Design | `OAuth` | No | [Go!](http://developer.dribbble.com/v1/) |
 | Noun Project | Icons | `OAuth` | No | [Go!](http://api.thenounproject.com/index.html) |
@@ -83,23 +79,20 @@ Please note a passing build status indicates all listed APIs are available since
 | Rijksmuseum| Art | `apiKey` | Yes | [Go!](https://www.rijksmuseum.nl/en/api) |
 
 ### Books
-
-| API | Description | Auth | HTTPS | Link |
+API | Description | Auth | HTTPS | Link |
 |---|---|---|---|---|
 | British National Bibliography | Books | No | No | [Go!](http://bnb.data.bl.uk/) |
 | Goodreads | Books | No | Yes | [Go!](https://www.goodreads.com/api) |
 | Google Books | Books | `OAuth` | Yes | [Go!](https://developers.google.com/books/) |
 
 ### Business
-
 | API | Description | Auth | HTTPS | Link |
 |---|---|---|---|---|
 | mailgun | Email Service | `apiKey` | Yes | [Go!](https://www.mailgun.com/) |
 | markerapi | Trademark Search | No | No | [Go!](http://www.markerapi.com/) |
 
 ### Calendar
-
-| API | Description | Auth | HTTPS | Link |
+API | Description | Auth | HTTPS | Link |
 |---|---|---|---|---|
 | Church Calendar | Catholic liturgical calendar | No | No | [Go!](http://calapi.inadiutorium.cz/) |
 | Holidays | Free API for obtaining information about holidays. | `apiKey` | Yes | [Go!](https://holidayapi.com/) |
@@ -107,8 +100,7 @@ Please note a passing build status indicates all listed APIs are available since
 | Non-Working Days | Database of ICS files for non working days | No | Yes | [Go!](https://github.com/gadael/icsdb) |
 
 ### Cloud Storage & File Sharing
-
-| API | Description | Auth | HTTPS | Link |
+API | Description | Auth | HTTPS | Link |
 |---|---|---|---|---|
 | Box | File Sharing and Storage | `OAuth` | Yes | [Go!](https://developer.box.com/) |
 | Dropbox | File Sharing and Storage | `OAuth` | Yes | [Go!](https://www.dropbox.com/developers) |
@@ -116,15 +108,13 @@ Please note a passing build status indicates all listed APIs are available since
 | OneDrive | File Sharing and Storage | `OAuth` | Yes | [Go!](https://dev.onedrive.com/) |
 
 ### Currency Exchange
-
-| API | Description | Auth | HTTPS | Link |
+API | Description | Auth | HTTPS | Link |
 |---|---|---|---|---|
 | Currencylayer | Exchange rates and currency conversion | `apiKey` | Yes | [Go!](https://currencylayer.com/documentation) |
 | Fixer.io | Exchange rates and currency conversion | No | Yes | [Go!](http://fixer.io) |
 
 ### Data Access
-
-| API | Description | Auth | HTTPS | Link |
+API | Description | Auth | HTTPS | Link |
 |---|---|---|---|---|
 | 18F | US Federal Government /Developer Program | No | No | [Go!](http://18f.github.io/API-All-the-X/) |
 | Abbreviation | Get abbreviations and meanings | `X-Mashape-Key` | Yes | [Go!](https://market.mashape.com/daxeel/abbreviations) |
@@ -151,8 +141,7 @@ Please note a passing build status indicates all listed APIs are available since
 | Yelp | Find Local Business | `OAuth` | Yes | [Go!](https://www.yelp.com/developers) |
 
 ### Data Validation
-
-| API | Description | Auth | HTTPS | Link |
+API | Description | Auth | HTTPS | Link |
 |---|---|---|---|---|
 | languagelayer | Language detection | No | Yes | [Go!](https://languagelayer.com) |
 | Lob.com | US Address Verification | `apiKey` | Yes | [Go!](https://lob.com/) |
@@ -161,8 +150,7 @@ Please note a passing build status indicates all listed APIs are available since
 | vatlayer | VAT number validation | No | Yes | [Go!](https://vatlayer.com) |
 
 ### Development
-
-| API | Description | Auth | HTTPS | Link |
+API | Description | Auth | HTTPS | Link |
 |---|---|---|---|---|
 | Adorable Avatars | Generate random cartoon avatars | No | Yes | [Go!](http://avatars.adorable.io) |
 | APIs.guru | Wikipedia for Web APIs, OpenAPI/Swagger specs for public APIs | No | Yes | [Go!](https://apis.guru/api-doc/) |
@@ -193,8 +181,7 @@ Please note a passing build status indicates all listed APIs are available since
 | Verse | Check what's the latest version of your favorite open-source project | No | Yes | [Go!](https://verse.pawelad.xyz/) |
 
 ### Documents & Productivity
-
-| API | Description | Auth | HTTPS |Link |
+API | Description | Auth | HTTPS |Link |
 |---|---|---|---|---|
 | File.io | File Sharing | No | Yes | [Go!](https://www.file.io) |
 | pdflayer API | HTML/URL to PDF | No | Yes | [Go!](https://pdflayer.com) |
@@ -203,23 +190,20 @@ Please note a passing build status indicates all listed APIs are available since
 | Wunderlist | Todo Lists | `OAuth` | Yes | [Go!](https://developer.wunderlist.com/documentation) |
 
 ### Environment
-
-| API | Description | Auth | HTTPS | Link |
+API | Description | Auth | HTTPS | Link |
 |---|---|---|---|---|
 | AirVisual | Air quality and weather data | `apiKey` | Yes | [Go!](https://airvisual.com/api) |
 | OpenAQ | Open air quality data | `apiKey` | Yes | [Go!](https://docs.openaq.org/) |
 | PM2.5.in | Air quality of China | `apiKey` | No | [Go!](http://www.pm25.in/api_doc) |
 
 ### Finance
-
-| API | Description | Auth | HTTPS | Link |
+API | Description | Auth | HTTPS | Link |
 |---|---|---|---|---|
 | Barchart OnDemand | Stock, Futures, and Forex Market Data | `apiKey` | Yes | [Go!](https://www.barchartondemand.com/free) |
 | Consumer Financial Protection Bureau | Financial services consumer complains data | `apiKey` | Yes | [Go!](https://data.consumerfinance.gov/resource/jhzv-w97w.json) |
 
 ### Food & Drink
-
-| API | Description | Auth | HTTPS | Link |
+API | Description | Auth | HTTPS | Link |
 |---|---|---|---|---|
 | BigOven | Recipe Search | `X-Mashape-Key` | No | [Go!](http://api2.bigoven.com/) |
 | BreweryDB | Beer | `apiKey` | No | [Go!](http://www.brewerydb.com/developers) |
@@ -234,8 +218,7 @@ Please note a passing build status indicates all listed APIs are available since
 
 
 ### Fraud Prevention
-
-| API | Description | Auth | HTTPS | Link |
+API | Description | Auth | HTTPS | Link |
 |---|---|---|---|---|
 | Whitepages Pro | Global identity verification with phone, address, email, and IP | `apiKey` | Yes | [Go!](https://pro.whitepages.com/developer/documentation/identity-check-api/) |
 | Whitepages Pro | Phone reputation to detect spammy phones | `apiKey` | Yes | [Go!](https://pro.whitepages.com/developer/documentation/phone-reputation-api/) |
@@ -244,8 +227,7 @@ Please note a passing build status indicates all listed APIs are available since
 | Whitepages Pro| Get normalized physical address, residents, address type, and validity. | `apiKey` | Yes | [Go!](https://pro.whitepages.com/developer/documentation/reverse-address-api/) |
 
 ### Games & Comics
-
-| API | Description | Auth | HTTPS | Link |
+API | Description | Auth | HTTPS | Link |
 |---|---|---|---|---|
 | Battle.net | Blizzard Entertainment | No | Yes | [Go!](https://dev.battle.net/)  |
 | Battlefield 4 | Battlefield 4 Information | No | Yes | [Go!](https://bf4stats.com/api)  |
@@ -267,8 +249,7 @@ Please note a passing build status indicates all listed APIs are available since
 | SWAPI | Star Wars Information | No | Yes | [Go!](https://swapi.co) |
 
 ### Geocoding
-
-| API | Description | Auth | HTTPS | Link |
+API | Description | Auth | HTTPS | Link |
 |---|---|---|---|---|
 | adresse.data.gouv.fr | Address database of France. geocoding and reverse. | No | Yes | [Go!](https://adresse.data.gouv.fr) |
 | Bing Maps | Create/customize digital maps based on Bing Maps data | `apiKey` | Yes | [Go!](https://www.microsoft.com/maps/) |
@@ -292,8 +273,7 @@ Please note a passing build status indicates all listed APIs are available since
 
 
 ### Health
-
-| API | Description | Auth | HTTPS | Link |
+API | Description | Auth | HTTPS | Link |
 |---|---|---|---|---|
 | Diabetes | Logging and retrieving diabetes information | No | No | [Go!](http://predictbgl.com/api/) |
 | Flutrack | Influenza-like symptoms with geotracking | No | No | [Go!](http://www.flutrack.org/) |
@@ -303,8 +283,7 @@ Please note a passing build status indicates all listed APIs are available since
 | USDA Nutrients | National Nutrient Database for Standard Reference | No | Yes | [Go!](https://ndb.nal.usda.gov/ndb/doc/index) |
 
 ### Machine Learning
-
-| API | Description | Auth | HTTPS | Link |
+API | Description | Auth | HTTPS | Link |
 |---|---|---|---|---|
 | API.AI | Natural Language Processing | `apikey` | Yes | [Go!](https://api.ai/) |
 | Clarifai | Computer Vision | `OAuth` | Yes | [Go!](https://developer.clarifai.com/) |
@@ -314,16 +293,14 @@ Please note a passing build status indicates all listed APIs are available since
 | Wit.ai | Natural Language Processing | `OAuth` | Yes | [Go!](https://wit.ai/) |
 
 ### Math
-
-| API | Description | Auth | HTTPS | Link |
+API | Description | Auth | HTTPS | Link |
 |---|---|---|---|---|
 | Newton | Symbolic and Arithmetic Math Calculator | No | Yes | [Go!](https://newton.now.sh/) |
 | Numbers API | Facts about numbers | No | No | [Go!](http://numbersapi.com) |
 
 
 ### Music
-
-| API | Description | Auth | HTTPS | Link |
+API | Description | Auth | HTTPS | Link |
 |---|---|---|---|---|
 | Bandsintown | Music Events | No | Yes | [Go!](https://app.swaggerhub.com/apis/Bandsintown/PublicAPI/3.0.0) |
 | Deezer | Music | `OAuth` | No | [Go!](http://developers.deezer.com/login?redirect=/api) |
@@ -342,24 +319,21 @@ Please note a passing build status indicates all listed APIs are available since
 | Vagalume | Crowdsourced lyrics and music knowledge | `apiKey` | Yes | [Go!](https://api.vagalume.com.br/docs/) |
 
 ### News
-
-| API | Description | Auth | HTTPS | Link |
+API | Description | Auth | HTTPS | Link |
 |---|---|---|---|---|
 | New York Times | Provides news | `apikey` | Yes | [Go!](https://developer.nytimes.com/) |
 | News API | headlines currently published on a range of news sources and blogs | `apikey` | Yes | [Go!](https://newsapi.org/) |
 | The Guardian | Access all the content the Guardian creates, categorised by tags and section | `apikey` | Yes | [Go!](http://open-platform.theguardian.com/) |
 
 ### Open Source projects
-
-| API | Description | Auth | HTTPS | Link |
+API | Description | Auth | HTTPS | Link |
 |---|---|---|---|---|
 | Countly  | Countly web analytics | No | No | [Go!](http://resources.count.ly/docs) |
 | Drupal.org | Drupal.org | No | Yes | [Go!](https://www.drupal.org/drupalorg/docs/api) |
 | Libraries.io | Open source software libraries | `apiKey` | Yes | [Go!](https://libraries.io/api) |
 
 ### Personality
-
-| API | Description | Auth | HTTPS | Link |
+API | Description | Auth | HTTPS | Link |
 |---|---|---|---|---|
 | chucknorris.io | JSON API for hand curated Chuck Norris jokes | No | Yes | [Go!](https://api.chucknorris.io) |
 | Forismatic | Inspirational Quotes | No | No | [Go!](http://forismatic.com/en/api/) |
@@ -369,8 +343,7 @@ Please note a passing build status indicates all listed APIs are available since
 | tronalddump.io | Api & web archive for the things Donald Trump has said | No | Yes | [Go!](https://www.tronalddump.io) |
 
 ### Photography
-
-| API | Description | Auth | HTTPS | Link |
+API | Description | Auth | HTTPS | Link |
 |---|---|---|---|---|
 | 500px |  Photography Community | `OAuth` | Yes | [Go!](https://github.com/500px/api-documentation) |
 | Flickr | Flickr Services | `OAuth` | Yes | [Go!](https://www.flickr.com/services/api/) |
@@ -381,8 +354,7 @@ Please note a passing build status indicates all listed APIs are available since
 | Unsplash | Photography | `OAuth` | Yes | [Go!](https://unsplash.com/developers) |
 
 ### Science
-
-| API | Description | Auth | HTTPS | Link |
+API | Description | Auth | HTTPS | Link |
 |---|---|---|---|---|
 | arcsecond.io | Multiple astronomy data sources | No | Yes | [Go!](https://api.arcsecond.io/) |
 | Fedger.io | Query machine intelligence data | No | Yes | [Go!](https://dev.fedger.io/docs/) |
@@ -396,21 +368,18 @@ Please note a passing build status indicates all listed APIs are available since
 | World Bank | World Data | No | No | [Go!](https://datahelpdesk.worldbank.org/knowledgebase/topics/125589) |
 
 ### Security
-
-| API | Description | Auth | HTTPS | Link |
+API | Description | Auth | HTTPS | Link |
 |---|---|---|---|---|
 | AXFR Database | AXFR public database | No | No | [Go!](http://api.axfrcheck.com) |
 | UK Police | UK Police data | No | Yes | [Go!](https://data.police.uk/docs/) |
 
 ### Shopping
-
-| API | Description | Auth | HTTPS | Link |
+API | Description | Auth | HTTPS | Link |
 |---|---|---|---|---|
 | eBay | Sell and Buy on eBay | `OAuth` | Yes | [Go!](https://go.developer.ebay.com/) |
 
 ### Social
-
-| API | Description | Auth | HTTPS | Link |
+API | Description | Auth | HTTPS | Link |
 |---|---|---|---|---|
 | Discord | Make bots for Discord, integrate Discord onto an external platform | `OAuth` | Yes | [Go!](https://discordapp.com/developers/docs/intro) |
 | DonReach Social Count | Get the social share count of a URL from every major social network | No | Yes | [Go!](https://donreach.com/social-share-count/) |
@@ -434,8 +403,7 @@ Please note a passing build status indicates all listed APIs are available since
 | vk | Read and write vk dat | `OAuth` | Yes | [Go!](https://vk.com/dev/sites) |
 
 ### Sports & Fitness
-
-| API | Description | Auth | HTTPS | Link |
+API | Description | Auth | HTTPS | Link |
 |---|---|---|---|---|
 | City Bikes | City Bikes around the world | No | No | [Go!](http://api.citybik.es/v2/) |
 | Ergast F1 | F1 data from the beginning of the world championships in 1950 | No | No | [Go!](http://ergast.com/mrd/) |
@@ -450,14 +418,12 @@ Please note a passing build status indicates all listed APIs are available since
 | Wger | Workout manager data as exercises, muscles or equipments | `apiKey` | Yes | [Go!](https://wger.de/en/software/api) |
 
 ### Tracking
-
-| API | Description | Auth | HTTPS | Link |
+API | Description | Auth | HTTPS | Link |
 |---|---|---|---|---|
 | Sweden  | Provides  information about parcels in transport | `apikey` | No | [Go!](https://developer.postnord.com/docs2) |
 
 ### Transportation
-
-| API | Description | Auth | HTTPS | Link |
+API | Description | Auth | HTTPS | Link |
 |---|---|---|---|---|
 | Amadeus Travel Innovation Sandbox | Travel Search - Limited usage | `apiKey` | Yes | [Go!](https://sandbox.amadeus.com/) |
 | Community Transit | Transitland API | No | Yes | [Go!](https://github.com/transitland/transitland-datastore/blob/master/README.md#api-endpoints) |
@@ -503,22 +469,19 @@ Please note a passing build status indicates all listed APIs are available since
 | WhereIsMyTransport | Platform for public transport data in emerging cities  | `OAuth` | Yes | [Go!](https://developer.whereismytransport.com/) |
 
 ### University
-
-| API | Description | Auth | HTTPS | Link |
+API | Description | Auth | HTTPS | Link |
 |---|---|---|---|---|
 | Universities List | University names, countries and domains| No | Yes | [Go!](https://github.com/Hipo/university-domains-list) |
 
 ### Vehicle
-
-| API | Description | Auth | HTTPS | Link |
+API | Description | Auth | HTTPS | Link |
 |---|---|---|---|---|
 | Vehicles | Lot of vehicles informations | `apiKey` | No | [Go!](http://developer.edmunds.com/api-documentation/overview/) |
 | Brazilian Vehicles and Prices | Vehicles information from Fundação Instituto de Pesquisas Econômicas - Fipe | No | Yes | [Go!](https://deividfortuna.github.io/fipe/) |
 | NHTSA Vehicles | NHTSA Product Information Catalog and Vehicle Listing | No | Yes | [Go!](https://vpic.nhtsa.dot.gov/api/) |
 
 ### Video
-
-| API | Description | Auth | HTTPS | Link |
+API | Description | Auth | HTTPS | Link |
 |---|---|---|---|---|
 | An API of Ice And Fire | Game Of Thrones API | No | Yes | [Go!](https://anapioficeandfire.com/) |
 | Dailymotion | Dailymotion Developer API | `OAuth` | Yes | [Go!](https://developer.dailymotion.com/) |
@@ -530,8 +493,7 @@ Please note a passing build status indicates all listed APIs are available since
 | YouTube | Add YouTube functionality to your sites and apps. | `OAuth` | Yes | [Go!](https://developers.google.com/youtube/) |
 
 ### Weather
-
-| API | Description | Auth | HTTPS | Link |
+API | Description | Auth | HTTPS | Link |
 |---|---|---|---|---|
 | Dark Sky | Weather | `apiKey` | Yes | [Go!](https://darksky.net/dev/) |
 | MetaWeather | Weather | No | Yes | [Go!](https://www.metaweather.com/api/) |

build/build.sh

+#!/bin/bash
+
+# create json directory if not already present
+mkdir -p ../json
+# parse API README and print (minified) JSON to stdout, redirect to /json
+node md2json.js ../README.md > ../json/api.min.json
+# beautify the previously created JSON file, redirect to /json
+python -m json.tool ../json/api.min.json > ../json/api.json
+# print out pretty JSON (useful for debugging, checking for new entries, etc) 
+cat ../json/api.json

build/deploy.sh

+#!/bin/bash
+
+set -o errexit -o nounset
+
+if [ "$TRAVIS_BRANCH" != "master" ]
+then
+  echo "This commit was made against $TRAVIS_BRANCH and not master! No deploy!"
+  exit 0
+fi
+
+rev=$(git rev-parse --short HEAD)
+
+mkdir deploy
+cd deploy
+
+git init
+git config --global user.name "Travis CI"
+git config --global user.email "builds@travis-ci.org"
+
+git remote add upstream "https://$GH_TOKEN@github.com/davemachado/public-apis.git"
+git fetch upstream
+git reset upstream/master
+
+mv ../../json .
+
+git add json/
+git commit -m "rebuild JSON at ${rev}"
+git push upstream HEAD:master
+

build/md2json.js

+fs = require('fs')
+
+function md_trim(str) {
+    return str.replace(/(^\s+)|(\s+$)/g, "");
+}
+
+function handle(filename, anchor) {
+    fs.readFile(filename, 'utf8', function (err,text) {
+      if (err) {
+        return console.log(err);
+      }
+    var lines = text.split("\n");
+    var cur_line = 0;
+    var line = ""
+    var table_name = "";
+    var col_num = 0;
+    var cols = [];
+    var rows = [];
+
+    function read_line() {
+        return lines[cur_line++];
+    }
+    var root = {};
+    while (true) {
+        var cols = [];
+        var rows = [];
+        while (line.indexOf(anchor) == -1 && cur_line != lines.length) {
+            line = read_line();
+        }
+        if (cur_line == lines.length) {
+            break;
+        }
+        table_name = line.split(anchor)[1];
+        table_name = md_trim(table_name)
+
+        line = read_line()
+
+        if (line) {
+            line = line.split("|")
+            for (var j in line) {
+
+                line[j] = md_trim(line[j])
+                if ((j == 0 || j == line.length - 1) && line[j] === "") {
+
+                } else {
+                    cols.push(line[j]);
+                }
+            }
+            if (line.length) {
+                cols = line;
+                rows.push(cols)
+            } else {
+                console.error("markdown expect column title")
+                break;
+            }
+        } else {
+            console.error("markdown expect table content")
+            break;
+        }
+
+        line = read_line()
+
+        if (line) {
+
+        } else {
+            console.error("markdown expect table spliter")
+            break;
+        }
+        line = read_line()
+        while (line.indexOf("|") != -1 && cur_line != lines.length) {
+
+            var line_this = line.split("|")
+            var row = []
+            for (var j in line_this) {
+                line_this[j] = md_trim(line_this[j])
+                if ((j == 0 || j == line_this.length - 1) && line_this[j] === "") {
+
+                } else {
+                    row.push(line_this[j]);
+                }
+
+            }
+            rows.push(row);
+            line = read_line()
+        }
+
+        var data=[];
+        for (var j in rows) {
+            if (j != 0) {
+                var ele = {};
+                for (var k in rows[j]) {
+                    ele[rows[0][k]] = rows[j][k];
+                }
+                data.push(ele);
+            }
+        }
+        root[table_name] = data;
+    }
+    console.log(JSON.stringify(root));
+  });
+}
+
+if (process.argv.length < 3) {
+    console.log("No .md file passed!");
+    return;
+}
+if (process.argv.length < 4) {
+  anchorText = "###";
+} else {
+  anchorText = process.argv[3];
+}
+handle(process.argv[2].toString(), anchorText);