Skip to content

submission #16

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 12 commits into
base: master
Choose a base branch
from
Open

submission #16

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
12 commits
Select commit Hold shift + click to select a range
3180122
part 1 script
EvanEzell Oct 15, 2018
f0833c7
Delete jdunca51.ipynb
EvanEzell Oct 15, 2018
422f223
remove trish file
EvanEzell Oct 22, 2018
9eae1d2
updated collection to my utk id
EvanEzell Oct 22, 2018
7de99f8
change id to my id
EvanEzell Oct 22, 2018
2e41ed8
create output file of urls with associated GH repos
EvanEzell Oct 22, 2018
d8cf142
changed id to my id
EvanEzell Oct 22, 2018
a2f0b3e
changed collection name to my collection
EvanEzell Oct 22, 2018
729a655
changed collection name to my collection
EvanEzell Oct 23, 2018
b818849
created list of releases
EvanEzell Oct 23, 2018
435fd66
fixed no common ancestor issue; output commits
EvanEzell Oct 24, 2018
8db3efd
changed filenames
EvanEzell Oct 29, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
93 changes: 19 additions & 74 deletions pgoedec1.ipynb → ...zell3-MiniProject2-Part1-checkpoint.ipynb
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,11 +1,16 @@
{
"cells": [
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### Retrieval for GitLab"
]
},
{
"cell_type": "code",
"execution_count": 1,
"metadata": {
"scrolled": false
},
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"import sys\n",
Expand All @@ -15,27 +20,22 @@
"import time\n",
"import datetime\n",
"import requests\n",
"from bs4 import BeautifulSoup\n",
"\n",
"dbname = \"fdac18mp2\" #please use this database\n",
"collname = \"tgoedecke\" #please modify so you store data in your collection\n",
"my_char = 'c'\n",
"\n",
"collname = \"glprj_eezell3\" #please modify so you store data in your collection\n",
"# beginning page index\n",
"begin = \"1\"\n",
"begin = \"0\"\n",
"client = pymongo.MongoClient()\n",
"\n",
"db = client[dbname]\n",
"coll = db[collname]\n",
"\n",
"\n",
"gitlab_url = \"https://gitlab.com/api/v4/projects?archived=false&membership=false&order_by=created_at&owned=false&page=\" + begin + \\\n",
"beginurl = \"https://gitlab.com/api/v4/projects?archived=false&membership=false&order_by=created_at&owned=false&page=\" + begin + \\\n",
" \"&per_page=99&simple=false&sort=desc&starred=false&statistics=false&with_custom_attributes=false&with_issues_enabled=false&with_merge_requests_enabled=false\"\n",
"\n",
"gleft = 20\n",
"\n",
"source_url = \"https://sourceforge.net/directory/?q=\" + my_char + \"&sort=name&page=\"\n",
"rest_url = \"https://sourceforge.net/rest/p/\"\n",
"gleft = 0\n",
"\n",
"header = {'per_page': 99}\n",
"\n",
Expand All @@ -49,47 +49,15 @@
" time .sleep(60)\n",
" return left\n",
"\n",
"def project_exists(url):\n",
" r = requests.get(url)\n",
" if r.status_code == 200:\n",
" return True\n",
" return False\n",
"\n",
"def get_source(url, coll, rest):\n",
" page = 1\n",
" project_count = 0\n",
" while True:\n",
" resp = requests.get(url + str(page))\n",
" text = resp.text\n",
" soup = BeautifulSoup(text, 'html.parser')\n",
" if re.search('No results found.', soup.get_text()):\n",
" return\n",
"\n",
" for link in soup.find_all(class_=\"project-icon\", href=True):\n",
" name = re.findall('/projects/([A-Za-z0-9\\-]*)', link.get('href'))\n",
" name = name[0] if name else None\n",
" if name is not None and name.lower().startswith(my_char):\n",
" resp = requests.get(rest + name)\n",
" if resp.status_code == 200:\n",
" info = json.loads(resp.text)\n",
" info['forge'] = 'sourceforge'\n",
" coll.insert_one(info)\n",
" project_count += 1\n",
" if project_count >= 50:\n",
" return\n",
" page += 1\n",
" return\n",
"\n",
"# send queries and extract urls \n",
"def get_gitlab(url, coll):\n",
"def get(url, coll):\n",
"\n",
" global gleft\n",
" global header\n",
" global bginnum\n",
" gleft = wait(gleft)\n",
" values = []\n",
" size = 0\n",
" project_count = 0\n",
"\n",
" try:\n",
" r = requests .get(url, headers=header)\n",
Expand All @@ -105,14 +73,8 @@
" array = json.loads(t)\n",
" \n",
" for el in array:\n",
" if el['name'].lower().startswith(my_char):\n",
" if project_exists(el['http_url_to_repo']):\n",
" project_count += 1\n",
" el['forge'] = 'gitlab'\n",
" coll.insert_one(el)\n",
" if project_count >= 50:\n",
" return\n",
" \n",
" coll.insert(el)\n",
" \n",
" #next page\n",
" while ('; rel=\"next\"' in lll):\n",
" gleft = int(r.headers.get('RateLimit-Remaining'))\n",
Expand All @@ -131,19 +93,13 @@
" t = r.text\n",
" array1 = json.loads(t)\n",
" for el in array1:\n",
" if el['name'].lower().startswith(my_char):\n",
" if project_exists(el['http_url_to_repo']):\n",
" project_count += 1\n",
" el['forge'] = 'gitlab'\n",
" coll.insert_one(el)\n",
" if project_count >= 50:\n",
" return\n",
" coll.insert(el)\n",
" else:\n",
" sys.stderr.write(\"url can not found:\\n\" + url + '\\n')\n",
" return \n",
" except requests.exceptions.ConnectionError:\n",
" sys.stderr.write('could not get ' + url + '\\n')\n",
" \n",
"\n",
" else:\n",
" sys.stderr.write(\"url can not found:\\n\" + url + '\\n')\n",
" return\n",
Expand All @@ -154,19 +110,8 @@
" sys.stderr.write(url + ';' + str(e) + '\\n')\n",
" \n",
"#start retrieving \n",
"get_gitlab(gitlab_url,coll)\n",
"get_source(source_url, coll, rest_url)\n",
"#print collected data\n",
"for doc in coll.find({}):\n",
" print(doc)"
"get(beginurl,coll)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
Expand Down
7 changes: 5 additions & 2 deletions compareRels.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@
headers = {'Accept': 'application/vnd.github.hellcat-preview+json'}

db = client['fdac18mp2'] # added in class
collName = 'releases_audris'
collName = 'releases_eezell3'
coll = db [collName]
def wait (left):
while (left < 20):
Expand Down Expand Up @@ -59,7 +59,10 @@ def cmp_rel (url):
v = get (url)
except Exception as e:
sys.stderr.write ("Could not get:" + url + ". Exception:" + str(e) + "\n")
print (url+';'+str(v['ahead_by'])+';'+str(v['behind_by']))
if 'ahead_by' in v and 'behind_by' in v:
print (url+';'+str(v['ahead_by'])+';'+str(v['behind_by']))
else:
sys.stderr.write ("Could not compare releases for: " + url + "; There exists no common ancestor between the two versions." + "\n")


p2r = {}
Expand Down
Loading