Skip to content
GitLab
Unverified Commit e9e6a71d authored by Dave Machado's avatar Dave Machado Committed by GitHub
Browse files

Merge pull request #569 from davemachado/update-md2json 

Update md2json, validate_format, and validate_links
parents f73086f4 b4521fc5
Hide whitespace changes
Inline Side-by-side
.travis.yml
View file @ e9e6a71d
language: node_js language: python
python: "3.6"
notifications: notifications:
email: true email: true
before_install:
- rvm install 2.4.0
install:
- gem install httparty ruby-progressbar
before_script: before_script:
- cd build - cd build
script: script:
......
build/build.sh
View file @ e9e6a71d
...@@ -3,7 +3,6 @@ ...@@ -3,7 +3,6 @@
# create json directory if not already present # create json directory if not already present
mkdir -p ../json mkdir -p ../json
# parse API README and print (minified) JSON to stdout, redirect to /json # parse API README and print (minified) JSON to stdout, redirect to /json
node condenseMd.js ../README.md > single_table.md ./md2json.py ../README.md > ../json/entries.min.json
node md2json.js single_table.md > ../json/entries.min.json
# beautify the previously created JSON file, redirect to /json # beautify the previously created JSON file, redirect to /json
python -m json.tool ../json/entries.min.json > ../json/entries.json python -m json.tool ../json/entries.min.json > ../json/entries.json
build/condenseMd.js deleted 100644 → 0
View file @ f73086f4
fs = require('fs')
function setupMd(filename, anchor) {
fs.readFile(filename, 'utf8', function (err,text) {
if (err) {
return console.log(err);
}
var lines = text.split("\n"),
cur_line = 0,
line = "",
table_name = "",
col_num = 0,
cols = [],
rows = [],
arr = [];
function read_line() {
return lines[cur_line++];
}
while (true) {
var cols = [];
var rows = [];
while (line.indexOf(anchor) == -1 && cur_line != lines.length) {
line = read_line();
}
if (cur_line == lines.length) {
break;
}
table_name = line.split(anchor)[1];
read_line()
read_line()
while (true) {
line = read_line()
if (line.length < 2 || cur_line == lines.length) {
break
}
if (line.indexOf("|") == 0) {
arr.push(line + table_name)
}
}
}
console.log(anchor + " entries")
console.log("API | Description | Auth | HTTPS | Link | Category")
console.log("|---|---|---|---|---|---|")
for (i = 0; i < arr.length; i++) {
console.log(arr[i])
}
});
}
if (process.argv.length < 3) {
console.log("No .md file passed!");
return;
}
if (process.argv.length < 4) {
anchorText = "###";
} else {
anchorText = process.argv[3];
}
setupMd(process.argv[2].toString(), anchorText);
build/main.sh
View file @ e9e6a71d
...@@ -30,7 +30,7 @@ else ...@@ -30,7 +30,7 @@ else
fi fi
echo "running format validation..." echo "running format validation..."
./validate_format.rb $FORMAT_FILE ./validate_format.py $FORMAT_FILE
if [[ $? != 0 ]]; then if [[ $? != 0 ]]; then
echo "format validation failed!" echo "format validation failed!"
exit 1 exit 1
......
build/md2json.js deleted 100644 → 0
View file @ f73086f4
fs = require('fs')
function md_trim(str, context) {
str = str.replace(/(^\s+)|(\s+$)/g, "");
if (context == 1) { // Name
// placeholder for any formatting on name value
} else if (context == 2) { // Description
str = str.replace(".", ""); // remove ending periods on descriptions
} else if (context == 3) { // Auth
if (str.toUpperCase() == "NO") {
str = null
} else {
str = str.replace("`", "").replace("`", "")
}
} else if (context == 4) { // HTTPS
if (str.toUpperCase() == "YES") {
str = true
} else {
str = false
}
} else if (context == 5) { // Link
str = str.replace("[Go!]", "").slice(1, -1);
}
return str;
}
function handle(filename, anchor) {
fs.readFile(filename, 'utf8', function (err,text) {
if (err) {
return console.log(err);
}
var lines = text.split("\n");
var cur_line = 0;
var line = ""
var table_name = "";
var col_num = 0;
var cols = [];
var rows = [];
var entry_count = 0;
function read_line() {
return lines[cur_line++];
}
var root = {};
while (true) {
var cols = [];
var rows = [];
while (line.indexOf(anchor) == -1 && cur_line != lines.length) {
line = read_line();
}
if (cur_line == lines.length) {
break;
}
table_name = line.split(anchor)[1];
table_name = md_trim(table_name, 0)
line = read_line()
if (line) {
line = line.split("|")
for (var j in line) {
line[j] = md_trim(line[j], 0)
if ((j == 0 || j == line.length - 1) && line[j] === "") {
} else {
cols.push(line[j]);
}
}
if (line.length) {
cols = line;
rows.push(cols)
} else {
console.error("markdown expect column title")
break;
}
} else {
console.error("markdown expect table content")
break;
}
line = read_line()
if (!line) {
console.error("markdown expect table spliter")
break;
}
line = read_line()
while (line.indexOf("|") != -1 && cur_line != lines.length) {
var line_this = line.split("|")
var row = []
for (var j in line_this) {
line_this[j] = md_trim(line_this[j], j)
if ((j == 0 || j == line_this.length - 1) && line_this[j] === "") {
} else {
row.push(line_this[j]);
}
}
rows.push(row);
entry_count++;
line = read_line()
}
var data=[];
for (var j in rows) {
if (j != 0) {
var ele = {};
for (var k in rows[j]) {
ele[rows[0][k]] = rows[j][k];
}
data.push(ele);
}
}
root["count"] = entry_count;
root[table_name] = data;
}
console.log(JSON.stringify(root));
});
}
if (process.argv.length < 3) {
console.log("No .md file passed!");
return;
}
if (process.argv.length < 4) {
anchorText = "###";
} else {
anchorText = process.argv[3];
}
handle(process.argv[2].toString(), anchorText);
build/md2json.py 0 → 100755
View file @ e9e6a71d
#!/usr/bin/env python3
import json
import sys
def markdown_to_json(filename, anchor):
"""Convert a Markdown file into a JSON string"""
category = ""
entries = []
with open(filename) as fp:
lines = (line.rstrip() for line in fp)
lines = list(line for line in lines if line \
and line.startswith(anchor) or line.startswith('| '))
for line in lines:
if line.startswith(anchor):
category = line.split(anchor)[1].strip()
continue
chunks = [x.strip() for x in line.split('|')[1:-1]]
entry = {
'API': chunks[0],
'Description': chunks[1],
'Auth': None if chunks[2].upper() == 'NO' else chunks[2].strip('`'),
'HTTPS': True if chunks[3].upper() == 'YES' else False,
'Link': chunks[4].replace('[Go!]', '')[1:-1],
'Category': category,
}
entries.append(entry)
final = {
'count': len(entries),
'entries': entries,
}
return json.dumps(final)
def main():
num_args = len(sys.argv)
if num_args < 2:
print("No .md file passed")
sys.exit(1)
if num_args < 3:
anchor = '###'
else:
anchor = sys.argv[2]
print(markdown_to_json(sys.argv[1], anchor))
if __name__ == "__main__":
main()
build/validate_format.py 0 → 100755
View file @ e9e6a71d
#!/usr/bin/env python3
import json
import string
import sys
anchor = '###'
auth_keys = ['apiKey', 'OAuth', 'X-Mashape-Key', 'No']
punctuation = ['.', '?', '!']
https_keys = ['Yes', 'No']
index_title = 0
index_desc = 1
index_auth = 2
index_https = 3
index_link = 4
errors = []
def add_error(line_num, message):
"""adds an error to the dynamic error list"""
err = '(L{:03d}) {}'.format(line_num+1, message)
errors.append(err)
def check_format(filename):
"""
validates that each line is formatted correctly,
appending to error list as needed
"""
with open(filename) as fp:
lines = list(line.rstrip() for line in fp)
# START Alphabetical Order
category = ""
sections = {}
section_line_num = {}
for line_num, line in enumerate(lines):
if line.startswith(anchor):
category = line.split(anchor)[1].strip()
sections[category] = []
section_line_num[category] = line_num
continue
if not line.startswith('|') or line.startswith('|---'):
continue
title = [x.strip() for x in line.split('|')[1:-1]][0].upper()
sections[category].append(title)
for category, entries in sections.items():
if sorted(entries) != entries:
add_error(section_line_num[category], '{} section is not in alphabetical order'.format(category))
# END Alphabetical Order
# START Check Entries
for line_num, line in enumerate(lines):
if not line.startswith('|') or line.startswith('|---'):
continue
segments = line.split('|')[1:-1]
# START Global
for segment in segments:
# every line segment should start and end with exactly 1 space
if len(segment) - len(segment.lstrip()) != 1 or len(segment) - len(segment.rstrip()) != 1:
add_error(line_num, "each segment must start and end with exactly 1 space")
# END Global
segments = [seg.strip() for seg in segments]
# START Description
# first character should be capitalized
char = segments[index_desc][0]
if char.upper() != char:
add_error(line_num, "first char of Description is not capitalized")
# last character should not punctuation
char = segments[index_desc][-1]
if char in punctuation:
add_error(line_num, "description should not end with {}".format(char))
# END Description
# START Auth
# values should conform to valid options only
auth = segments[index_auth].replace('`', '')
if auth not in auth_keys:
add_error(line_num, "{} is not a valid Auth option".format(auth))
# END Auth
# START HTTPS
# values should conform to valid options only
https = segments[index_https]
if https not in https_keys:
add_error(line_num, "{} is not a valid HTTPS option".format(https))
# END HTTPS
# START Link
# url should be wrapped in '[Go!]()' Markdown syntax
link = segments[index_link]
if not link.startswith('[Go!](http') or not link.endswith(')'):
add_error(line_num, 'link format should be "[Go!](LINK)"')
# END Link
# END Check Entries
def main():
num_args = len(sys.argv)
if num_args < 2:
print("No .md file passed")
sys.exit(1)
check_format(sys.argv[1])
if len(errors) > 0:
for err in errors:
print(err)
sys.exit(1)
if __name__ == "__main__":
main()
build/validate_format.rb deleted 100755 → 0
View file @ f73086f4
#!/usr/bin/env ruby
auth_keys = ['apiKey', 'OAuth', 'X-Mashape-Key', 'No']
punctuation = ['.', '?', '!']
https_keys = ['Yes', 'No']
INDEX_TITLE = 1
INDEX_DESCRIPTION = 2
INDEX_AUTH = 3
INDEX_HTTPS = 4
INDEX_LINK = 5
filename = ARGV[0]
$errors = []
def add_error(line_num, val_index, message)
case val_index
when INDEX_TITLE
segment = "Title"
when INDEX_DESCRIPTION
segment = "Description"
when INDEX_AUTH
segment = "Auth"
when INDEX_HTTPS
segment = "HTTPS"
when INDEX_LINK
segment = "Link"
end
$errors.push("(L%03d) %-14.14s #{message}" % [line_num, segment])
end
################### CHECK ALPHABETICAL ORDER ###################
section = ''
sections = []
section_to_line_num = {}
section_to_entries = Hash.new {|h,k| h[k] = Array.new }
File.foreach(filename).with_index do | line, line_num |
if line.start_with?('###')
section = line.sub('###', '').lstrip.chop
sections.push(section)
section_to_line_num[section] = line_num + 1
end
# Skip non-markdown table lines and table schema lines
if !line.start_with?('|') || line.eql?("|---|---|---|---|---|\n")
next
end
# char to check is the first column
check_char = line.split("|")[1].strip.upcase
section_to_entries[section].push(check_char)
end
sections.each do | sect |
if section_to_entries[sect] != section_to_entries[sect].sort
add_error(section_to_line_num[sect], INDEX_TITLE, "#{sect} section is not in alphabetical order")
end
end
#################### CHECK LINE ENTRIES ########################
File.foreach(filename).with_index do | line, line_num |
line_num += 1
# Skip non-markdown table lines and table schema lines
if !line.start_with?('|') || line.eql?("|---|---|---|---|---|\n")
next
end
values = line.split("|")
################### GLOBAL ###################
values.each.with_index do |val, val_index|
msg = ""
case val_index
when INDEX_TITLE..INDEX_LINK
# every line segment should start and end with exactly 1 space
if val[/\A */].size != 1 || val[/ *\z/].size != 1
add_error(line_num, val_index, "string should start and end with exactly 1 space")
end
end
end
################# DESCRIPTION ################
# First character should be capitalized
desc_val = values[INDEX_DESCRIPTION].lstrip.chop
if !/[[:upper:]]/.match(desc_val[0])
add_error(line_num, INDEX_DESCRIPTION, "first char not uppercase")
end
# value should not be punctuated
last_char = desc_val[desc_val.length-1]
if punctuation.include?(last_char)
add_error(line_num, INDEX_DESCRIPTION, "description should not end with \"#{last_char}\"")
end
#################### AUTH ####################
# Values should conform to valid options only
auth_val = values[INDEX_AUTH].lstrip.chop.tr('``', '')
if !auth_keys.include?(auth_val)
add_error(line_num, INDEX_AUTH, "not a valid option: #{auth_val}")
end
#################### HTTPS ###################
# Values should be either "Yes" or "No"
https_val = values[INDEX_HTTPS].lstrip.chop
if !https_keys.include?(https_val)
add_error(line_num, INDEX_HTTPS, "must use \"Yes\" or \"No\": #{https_val}")
end
#################### LINK ####################
# Url should be wrapped in "[Go!]" view
link_val = values[INDEX_LINK].lstrip.chop
if !link_val.start_with?("[Go!](") || !link_val.end_with?(')')
add_error(line_num, INDEX_LINK, "format should be \"[Go!](<LINK>)\": #{link_val}")
end
end
$errors.each do | e |
puts e
end
exit($errors.length)
build/validate_links.py 0 → 100755
View file @ e9e6a71d
#!/usr/bin/env python3
import httplib2
import json
import socket
import sys
def parse_links(filename):
"""Returns a list of links from JSON object"""
data = json.load(open(filename))
links = []
for entry in data['entries']:
link = entry['Link']
https = True if link.startswith('https') else False
x = {
'link': link,
'https': https,
}
links.append(x)
return links
def validate_links(links):
"""Checks each entry in JSON file for live link"""
print('Validating {} links...'.format(len(links)))
errors = []
for each in links:
link = each['link']
h = httplib2.Http(disable_ssl_certificate_validation=True, timeout=5)
try:
resp = h.request(link, 'HEAD')
code = int(resp[0]['status'])
# check if status code is a client or server error
if code >= 404:
errors.append('{}: {}'.format(code, link))
except TimeoutError:
errors.append("TMO: " + link)
except socket.error as socketerror:
errors.append("SOC: {} : {}".format(socketerror, link))
return errors
if __name__ == "__main__":
num_args = len(sys.argv)
if num_args < 2:
print("No .json file passed")
sys.exit(1)
errors = validate_links(parse_links(sys.argv[1]))
if len(errors) > 0:
for err in errors:
print(err)
sys.exit(1)
build/validate_links.rb deleted 100755 → 0
View file @ f73086f4
#!/usr/bin/env ruby
require 'httparty'
require 'ruby-progressbar'
require 'uri'
allowed_codes = [200, 302, 403, 429]
allowed_links = ["https://www.yelp.com/developers/documentation/v3"]
args = ARGV
filename = args[0]
contents = File.open(filename, 'rb') { |f| f.read }
raw_links = URI.extract(contents, ['http', 'https'])
# Remove trailing ')' from entry URLs
links = []
raw_links.each do |link|
if link.end_with?(')')
links.push(link[0...-1])
else
links.push(link)
end
end
if links.length <= 0
puts "no links to check"
exit(0)
end
fails = []
# Fail on any duplicate elements
dup = links.select{|element| links.count(element) > 1}
if dup.uniq.length > 0
dup.uniq.each do |e|
fails.push("(DUP): #{e}")
end
end
# Remove any duplicates from array
links = links.uniq
count = 0
total = links.length
progressbar = ProgressBar.create(:total => total,
:format => "%a %P% | Processed: %c from %C")
# GET each link and check for valid response code from allowed_codes
links.each do |link|
begin
count += 1
if allowed_links.include?(link)
next
end
res = HTTParty.get(link, timeout: 10)
if res.code.nil?
fails.push("(NIL): #{link}")
next
end
if !allowed_codes.include?(res.code)
fails.push("(#{res.code}): #{link}")
end
rescue HTTParty::RedirectionTooDeep
fails.push("(RTD): #{link}")
rescue Net::ReadTimeout
fails.push("(TMO): #{link}")
rescue Net::OpenTimeout
fails.push("(TMO): #{link}")
rescue OpenSSL::SSL::SSLError
fails.push("(SSL): #{link}")
rescue SocketError
fails.push("(SOK): #{link}")
rescue Errno::ECONNREFUSED
fails.push("(CON): #{link}")
rescue Errno::ECONNRESET
next
end
progressbar.increment
end
puts "#{count}/#{total} links checked"
if fails.length <= 0
puts "all links valid"
exit(0)
else
puts "-- RESULTS --"
fails.sort!
fails.each do |e|
puts e
end
exit(1)
end
json/entries.json
View file @ e9e6a71d
...@@ -197,7 +197,7 @@ ...@@ -197,7 +197,7 @@
"API": "BookNomads", "API": "BookNomads",
"Auth": null, "Auth": null,
"Category": "Books", "Category": "Books",
"Description": "Books published in the Netherlands and Flanders (about 25 million), book covers, and related data", "Description": "Books published in the Netherlands and Flanders (about 2.5 million), book covers, and related data",
"HTTPS": true, "HTTPS": true,
"Link": "https://www.booknomads.com/dev" "Link": "https://www.booknomads.com/dev"
}, },
...@@ -765,7 +765,7 @@ ...@@ -765,7 +765,7 @@
"API": "JSONbin.io", "API": "JSONbin.io",
"Auth": "apiKey", "Auth": "apiKey",
"Category": "Development", "Category": "Development",
"Description": "Free JSON storage service Ideal for small scale Web apps, Websites and Mobile apps", "Description": "Free JSON storage service. Ideal for small scale Web apps, Websites and Mobile apps",
"HTTPS": true, "HTTPS": true,
"Link": "https://jsonbin.io" "Link": "https://jsonbin.io"
}, },
...@@ -1741,7 +1741,7 @@ ...@@ -1741,7 +1741,7 @@
"API": "BusinessUSA", "API": "BusinessUSA",
"Auth": "apiKey", "Auth": "apiKey",
"Category": "Government", "Category": "Government",
"Description": "BusinessUSA gives developers access to authoritative information on US. programs, events, services and more", "Description": "BusinessUSA gives developers access to authoritative information on U.S. programs, events, services and more",
"HTTPS": true, "HTTPS": true,
"Link": "https://business.usa.gov/developer" "Link": "https://business.usa.gov/developer"
}, },
...@@ -1781,7 +1781,7 @@ ...@@ -1781,7 +1781,7 @@
"API": "Regulations.gov", "API": "Regulations.gov",
"Auth": "apiKey", "Auth": "apiKey",
"Category": "Government", "Category": "Government",
"Description": "Regulationsgov provides access to Federal regulatory materials and increases public participation and their understanding of the Federal rule making process", "Description": "Regulations.gov provides access to Federal regulatory materials and increases public participation and their understanding of the Federal rule making process",
"HTTPS": true, "HTTPS": true,
"Link": "https://regulationsgov.github.io/developers/" "Link": "https://regulationsgov.github.io/developers/"
}, },
...@@ -1837,7 +1837,7 @@ ...@@ -1837,7 +1837,7 @@
"API": "Medicare", "API": "Medicare",
"Auth": null, "Auth": null,
"Category": "Health", "Category": "Health",
"Description": "Access to the data from the CMS - medicaregov", "Description": "Access to the data from the CMS - medicare.gov",
"HTTPS": true, "HTTPS": true,
"Link": "https://data.medicare.gov/developers" "Link": "https://data.medicare.gov/developers"
}, },
...@@ -2477,7 +2477,7 @@ ...@@ -2477,7 +2477,7 @@
"API": "UPC database", "API": "UPC database",
"Auth": "apiKey", "Auth": "apiKey",
"Category": "Open Data", "Category": "Open Data",
"Description": "More than 15 million barcode numbers from all around the world", "Description": "More than 1.5 million barcode numbers from all around the world",
"HTTPS": true, "HTTPS": true,
"Link": "https://upcdatabase.org/api" "Link": "https://upcdatabase.org/api"
}, },
...@@ -2517,7 +2517,7 @@ ...@@ -2517,7 +2517,7 @@
"API": "Drupal.org", "API": "Drupal.org",
"Auth": null, "Auth": null,
"Category": "Open Source Projects", "Category": "Open Source Projects",
"Description": "Drupalorg", "Description": "Drupal.org",
"HTTPS": true, "HTTPS": true,
"Link": "https://www.drupal.org/drupalorg/docs/api" "Link": "https://www.drupal.org/drupalorg/docs/api"
}, },
...@@ -2733,7 +2733,7 @@ ...@@ -2733,7 +2733,7 @@
"API": "inspirehep.net", "API": "inspirehep.net",
"Auth": null, "Auth": null,
"Category": "Science", "Category": "Science",
"Description": "High Energy Physics info system", "Description": "High Energy Physics info. system",
"HTTPS": true, "HTTPS": true,
"Link": "https://inspirehep.net/info/hep/api?ln=en" "Link": "https://inspirehep.net/info/hep/api?ln=en"
}, },
...@@ -2749,7 +2749,7 @@ ...@@ -2749,7 +2749,7 @@
"API": "Minor Planet Center", "API": "Minor Planet Center",
"Auth": null, "Auth": null,
"Category": "Science", "Category": "Science",
"Description": "Asterankcom Information", "Description": "Asterank.com Information",
"HTTPS": false, "HTTPS": false,
"Link": "http://www.asterank.com/mpc" "Link": "http://www.asterank.com/mpc"
}, },
...@@ -2949,7 +2949,7 @@ ...@@ -2949,7 +2949,7 @@
"API": "Meetup.com", "API": "Meetup.com",
"Auth": "apiKey", "Auth": "apiKey",
"Category": "Social", "Category": "Social",
"Description": "Data about Meetups from Meetupcom", "Description": "Data about Meetups from Meetup.com",
"HTTPS": true, "HTTPS": true,
"Link": "https://www.meetup.com/meetup_api/" "Link": "https://www.meetup.com/meetup_api/"
}, },
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment