diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md
index 753f97da..bf1a4b79 100644
--- a/.github/ISSUE_TEMPLATE/feature_request.md
+++ b/.github/ISSUE_TEMPLATE/feature_request.md
@@ -1,6 +1,6 @@
---
name: Feature request
-about: Idea or feature request
+about: Idea or feature request
title: ''
labels: 'kind:feature'
assignees: ''
diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
new file mode 100644
index 00000000..f7f27c0f
--- /dev/null
+++ b/.github/workflows/ci.yaml
@@ -0,0 +1,30 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+---
+name: CI
+on:
+ - push
+ - pull_request
+
+jobs:
+ statics:
+ name: Static Checks
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@v2
+ - uses: actions/setup-python@v1
+ - uses: pre-commit/action@v1.0.1
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 07fae567..3dd6d750 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -28,4 +28,6 @@ repos:
hooks:
- id: check-yaml
- id: end-of-file-fixer
+ exclude: ^ui/vendors/.*$
- id: trailing-whitespace
+ exclude: ^ui/vendors/.*$
diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md
index 5fd7277c..d5c66bb2 100644
--- a/CODE_OF_CONDUCT.md
+++ b/CODE_OF_CONDUCT.md
@@ -1,7 +1,7 @@
# Code of Conduct
-The Apache Kibble project follows the
+The Apache Kibble project follows the
[Apache Software Foundation code of conduct](https://www.apache.org/foundation/policies/conduct.html).
-If you observe behavior that violates those rules please follow the
+If you observe behavior that violates those rules please follow the
[ASF reporting guidelines](https://www.apache.org/foundation/policies/conduct#reporting-guidelines).
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 86ec40be..2c3351a7 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -1,6 +1,6 @@
# Contributing to Kibble #
-## Community
+## Community
The main development and design discussion happens on our mailing lists.
We have a list specifically for development, and one for future user questions and feedback.
@@ -15,5 +15,31 @@ We also have:
## Development installation
-This project requires Python in higher version than 3.3.
+This project requires Python in higher version than 3.4.
More information will come soon!
+
+## Code Quality
+
+Apache Kibble project is using [pre-commits](https://pre-commit.com) to ensure the quality of the code.
+We encourage you to use pre-commits, but it's not required in order to contribute. Every change is checked
+on CI and if it does not pass the tests it cannot be accepted. If you want to check locally then
+you should install Python3.6 or newer together and run:
+```bash
+pip install pre-commit
+# or
+brew install pre-commit
+```
+For more installation options visit the [pre-commits](https://pre-commit.com).
+
+To turn on pre-commit checks for commit operations in git, run:
+```bash
+pre-commit install
+```
+To run all checks on your staged files, run:
+```bash
+pre-commit run
+```
+To run all checks on all files, run:
+```bash
+pre-commit run --all-files
+```
diff --git a/NOTICE b/NOTICE
index 790683a8..66e7dc12 100644
--- a/NOTICE
+++ b/NOTICE
@@ -119,7 +119,7 @@ CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
Metis Dashboard (MIT License)
------------------------------------------------------------------------
-Copyright (c) 2015 onokumus
+Copyright (c) 2015 onokumus
Permission is hereby granted, free of charge, to any person obtaining a copy of
this software and associated documentation files (the "Software"), to deal in
the Software without restriction, including without limitation the rights to
@@ -186,4 +186,3 @@ IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
-
diff --git a/README.md b/README.md
index 7efe0455..3d9f3728 100644
--- a/README.md
+++ b/README.md
@@ -1,6 +1,8 @@

# Apache Kibble
+
+[](http://www.apache.org/licenses/LICENSE-2.0.txt)
Apache Kibble is a tool to collect, aggregate and visualize data about any software project that uses commonly known tools. It consists of two components:
@@ -8,7 +10,7 @@ Apache Kibble is a tool to collect, aggregate and visualize data about any softw
for the scanners to connect to, and provides the overall management of sources as well as the
visualizations and API end points.
- **Kibble scanners** ([kibble-scanners](https://github.com/apache/kibble-scanners)) - a collection of
- scanning applications each designed to work with a specific type of resource (git repo, mailing list,
+ scanning applications each designed to work with a specific type of resource (git repo, mailing list,
JIRA, etc) and push compiled data objects to the Kibble Server.
## Documentation
diff --git a/api/handler.py b/api/handler.py
index d767e692..41663a53 100644
--- a/api/handler.py
+++ b/api/handler.py
@@ -56,7 +56,7 @@ class KibbleHTTPError(Exception):
def __init__(self, code, message):
self.code = code
self.message = message
-
+
class KibbleAPIWrapper:
"""
@@ -67,7 +67,7 @@ def __init__(self, path, func):
self.API = KibbleOpenAPI
self.path = path
self.exception = KibbleHTTPError
-
+
def __call__(self, environ, start_response, session):
"""Run the function, return response OR return stacktrace"""
response = None
@@ -90,7 +90,7 @@ def __call__(self, environ, start_response, session):
"reason": "Invalid JSON: %s" % err
})
return
-
+
# Validate URL against OpenAPI specs
try:
self.API.validate(environ['REQUEST_METHOD'], self.path, formdata)
@@ -102,7 +102,7 @@ def __call__(self, environ, start_response, session):
"reason": err.message
})
return
-
+
# Call page with env, SR and form data
try:
response = self.func(self, environ, formdata, session)
@@ -124,7 +124,7 @@ def __call__(self, environ, start_response, session):
"reason": err.message
}, indent = 4) + "\n"
return
-
+
except:
err_type, err_value, tb = sys.exc_info()
traceback_output = ['API traceback:']
@@ -140,8 +140,8 @@ def __call__(self, environ, start_response, session):
"code": "500",
"reason": '\n'.join(traceback_output)
})
-
-
+
+
def fourohfour(environ, start_response):
"""A very simple 404 handler"""
start_response("404 Not Found", [
@@ -181,7 +181,7 @@ def application(environ, start_response):
elif isinstance(bucket, bytes):
yield bucket
return
-
+
for bucket in fourohfour(environ, start_response):
yield bytes(bucket, encoding = 'utf-8')
diff --git a/api/pages/__init__.py b/api/pages/__init__.py
index 67a30574..1f8ef384 100644
--- a/api/pages/__init__.py
+++ b/api/pages/__init__.py
@@ -42,5 +42,5 @@ def loadPage(path):
xp = p.replace('.', '/')
print("Loading endpoint pages.%s as %s" % (p, xp))
handlers[xp] = importlib.import_module("pages.%s" % p)
-
-loadPage(rootpath)
\ No newline at end of file
+
+loadPage(rootpath)
diff --git a/api/pages/account.py b/api/pages/account.py
index 48577735..c3d700bd 100644
--- a/api/pages/account.py
+++ b/api/pages/account.py
@@ -85,7 +85,7 @@
# $ref: '#/components/schemas/Error'
# description: unexpected error
# summary: Create a new account
-#
+#
########################################################################
@@ -125,7 +125,7 @@ def sendCode(session, addr, code):
s.quit()
def run(API, environ, indata, session):
-
+
method = environ['REQUEST_METHOD']
# Add a new account??
@@ -133,32 +133,32 @@ def run(API, environ, indata, session):
u = indata['email']
p = indata['password']
d = indata['displayname']
-
+
# Are new accounts allowed? (admin can always make accounts, of course)
if not session.config['accounts'].get('allowSignup', False):
if not (session.user and session.user['level'] == 'admin'):
raise API.exception(403, "New account requests have been administratively disabled.")
-
+
# Check if we already have that username in use
if session.DB.ES.exists(index=session.DB.dbname, doc_type='useraccount', id = u):
raise API.exception(403, "Username already in use")
-
+
# We require a username, displayName password of at least 3 chars each
if len(p) < 3 or len(u) < 3 or len(d) < 3:
raise API.exception(400, "Username, display-name and password must each be at elast 3 characters long.")
-
+
# We loosely check that the email is an email
if not re.match(r"^\S+@\S+\.\S+$", u):
raise API.exception(400, "Invalid email address presented.")
-
+
# Okay, let's make an account...I guess
salt = bcrypt.gensalt()
pwd = bcrypt.hashpw(p.encode('utf-8'), salt).decode('ascii')
-
+
# Verification code, if needed
vsalt = bcrypt.gensalt()
vcode = hashlib.sha1(vsalt).hexdigest()
-
+
# Auto-verify unless verification is enabled.
# This is so previously unverified accounts don'thave to verify
# if we later turn verification on.
@@ -167,7 +167,7 @@ def run(API, environ, indata, session):
verified = False
sendCode(session, u, vcode) # Send verification email
# If verification email fails, skip account creation.
-
+
doc = {
'email': u, # Username (email)
'password': pwd, # Hashed password
@@ -179,24 +179,24 @@ def run(API, environ, indata, session):
'vcode': vcode, # Verification code
'userlevel': "user" # User level (user/admin)
}
-
-
+
+
# If we have auto-invite on, check if there are orgs to invite to
if 'autoInvite' in session.config['accounts']:
dom = u.split('@')[-1].lower()
for ai in session.config['accounts']['autoInvite']:
if ai['domain'] == dom:
doc['organisations'].append(ai['organisation'])
-
+
session.DB.ES.index(index=session.DB.dbname, doc_type='useraccount', id = u, body = doc)
yield json.dumps({"message": "Account created!", "verified": verified})
return
-
+
# We need to be logged in for the rest of this!
if not session.user:
raise API.exception(403, "You must be logged in to use this API endpoint! %s")
-
-
+
+
# Patch (edit) an account
if method == "PATCH":
userid = session.user['email']
@@ -217,4 +217,3 @@ def run(API, environ, indata, session):
session.DB.ES.index(index=session.DB.dbname, doc_type='useraccount', id = userid, body = udoc)
yield json.dumps({"message": "Account updated!"})
return
-
\ No newline at end of file
diff --git a/api/pages/bio/bio.py b/api/pages/bio/bio.py
index 0f43f9a3..c3aa5338 100644
--- a/api/pages/bio/bio.py
+++ b/api/pages/bio/bio.py
@@ -56,7 +56,7 @@
# security:
# - cookieAuth: []
# summary: Shows some facts about a contributor
-#
+#
########################################################################
@@ -72,30 +72,30 @@
import hashlib
def run(API, environ, indata, session):
-
+
# We need to be logged in for this!
if not session.user:
raise API.exception(403, "You must be logged in to use this API endpoint! %s")
-
+
now = time.time()
-
+
# First, fetch the view if we have such a thing enabled
viewList = []
if indata.get('view'):
viewList = session.getView(indata.get('view'))
if indata.get('subfilter'):
- viewList = session.subFilter(indata.get('subfilter'), view = viewList)
-
-
+ viewList = session.subFilter(indata.get('subfilter'), view = viewList)
+
+
dOrg = session.user['defaultOrganisation'] or "apache"
-
+
pid = hashlib.sha1( ("%s%s" % (dOrg, indata.get('email', '???'))).encode('ascii', errors='replace')).hexdigest()
person = {}
if session.DB.ES.exists(index=session.DB.dbname, doc_type="person", id = pid):
person = session.DB.ES.get(index=session.DB.dbname, doc_type="person", id = pid)['_source']
else:
raise API.exception(404, "No such biography!")
-
+
query = {
'query': {
'bool': {
@@ -125,8 +125,8 @@ def run(API, environ, indata, session):
{'term': {codeKey: indata.get('email')}},
]
query['query']['bool']['minimum_should_match'] = 1
-
-
+
+
# FIRST EMAIL
res = session.DB.ES.search(
index=session.DB.dbname,
@@ -136,7 +136,7 @@ def run(API, environ, indata, session):
firstEmail = None
if res['hits']['hits']:
firstEmail = res['hits']['hits'][0]['_source']['ts']
-
+
# FIRST COMMIT
res = session.DB.ES.search(
index=session.DB.dbname,
@@ -146,7 +146,7 @@ def run(API, environ, indata, session):
firstCommit = None
if res['hits']['hits']:
firstCommit = res['hits']['hits'][0]['_source']['ts']
-
+
# FIRST AUTHORSHIP
query['query']['bool']['should'][3] = {'term': {'author_email': indata.get('email')}}
res = session.DB.ES.search(
@@ -157,8 +157,8 @@ def run(API, environ, indata, session):
firstAuthor = None
if res['hits']['hits']:
firstAuthor = res['hits']['hits'][0]['_source']['ts']
-
-
+
+
# COUNT EMAIL, CODE, LINES CHANGED
del query['sort']
del query['size']
@@ -167,13 +167,13 @@ def run(API, environ, indata, session):
doc_type="email",
body = query
)['count']
-
+
no_commits = session.DB.ES.count(
index=session.DB.dbname,
doc_type="code_commit",
body = query
)['count']
-
+
JSON_OUT = {
'found': True,
'bio': {
diff --git a/api/pages/bio/newtimers.py b/api/pages/bio/newtimers.py
index 12245f16..8dd4dc20 100644
--- a/api/pages/bio/newtimers.py
+++ b/api/pages/bio/newtimers.py
@@ -56,7 +56,7 @@
# security:
# - cookieAuth: []
# summary: Shows some facts about a contributor
-#
+#
########################################################################
@@ -75,7 +75,7 @@ def find_earlier(session, query, when, who, which, where, doctype, dOrg):
"""Find earlier document pertaining to this user. return True if found"""
if 'aggs' in query:
del query['aggs']
-
+
rangeQuery = {'range':
{
which: {
@@ -84,7 +84,7 @@ def find_earlier(session, query, when, who, which, where, doctype, dOrg):
}
}
}
-
+
query['query']['bool']['must'] = [
rangeQuery,
{
@@ -96,12 +96,12 @@ def find_earlier(session, query, when, who, which, where, doctype, dOrg):
'term': {
where: who
}
-
+
}
]
query['size'] = 1
query['sort'] = [{ which: 'asc' }]
-
+
res = session.DB.ES.search(
index=session.DB.dbname,
doc_type=doctype,
@@ -115,40 +115,40 @@ def find_earlier(session, query, when, who, which, where, doctype, dOrg):
return [-1, None]
else:
return [-1, None]
-
+
def run(API, environ, indata, session):
-
+
# We need to be logged in for this!
if not session.user:
raise API.exception(403, "You must be logged in to use this API endpoint! %s")
-
+
now = time.time()
-
+
# First, fetch the view if we have such a thing enabled
viewList = []
if indata.get('view'):
viewList = session.getView(indata.get('view'))
if indata.get('subfilter'):
- viewList = session.subFilter(indata.get('subfilter'), view = viewList)
-
-
+ viewList = session.subFilter(indata.get('subfilter'), view = viewList)
+
+
dOrg = session.user['defaultOrganisation'] or "apache"
-
-
+
+
# Keep track of all contributors, and newcomers
contributors = []
newcomers = {}
-
+
####################################################################
# Start by grabbing all contributors this period via terms agg #
####################################################################
dateTo = indata.get('to', int(time.time()))
dateFrom = indata.get('from', dateTo - (86400*30*6)) # Default to a 6 month span
-
-
-
-
+
+
+
+
############################
# CODE NEWTIMERS #
############################
@@ -161,7 +161,7 @@ def run(API, environ, indata, session):
}
}
}
-
+
query = {
'query': {
'bool': {
@@ -176,46 +176,46 @@ def run(API, environ, indata, session):
}
}
}
-
+
query['aggs'] = {
'by_committer': {
'terms': {
'field': 'committer_email',
'size': 500
- }
+ }
},
'by_author': {
'terms': {
'field': 'author_email',
'size': 500
- }
+ }
}
}
-
+
# Source-specific or view-specific??
if indata.get('source'):
query['query']['bool']['must'].append({'term': {'sourceID': indata.get('source')}})
elif viewList:
query['query']['bool']['must'].append({'terms': {'sourceID': viewList}})
-
-
+
+
res = session.DB.ES.search(
index=session.DB.dbname,
doc_type="code_commit",
body = query
)
-
+
code_contributors = []
for bucket in res['aggregations']['by_committer']['buckets']:
email = bucket['key']
if email not in code_contributors:
code_contributors.append(email)
-
+
for bucket in res['aggregations']['by_author']['buckets']:
email = bucket['key']
if email not in code_contributors:
code_contributors.append(email)
-
+
# Now, for each contributor, find if they have done anything before
for email in code_contributors:
ea = find_earlier(session, query, dateFrom, email, 'ts', 'author_email', 'code_commit', dOrg)
@@ -227,9 +227,9 @@ def run(API, environ, indata, session):
newcomers[email] = {
'code': earliest
}
-
-
-
+
+
+
############################
# ISSUE NEWTIMERS #
############################
@@ -242,7 +242,7 @@ def run(API, environ, indata, session):
}
}
}
-
+
query = {
'query': {
'bool': {
@@ -257,46 +257,46 @@ def run(API, environ, indata, session):
}
}
}
-
+
query['aggs'] = {
'by_creator': {
'terms': {
'field': 'issueCreator',
'size': 500
- }
+ }
},
'by_closer': {
'terms': {
'field': 'issueCloser',
'size': 500
- }
+ }
}
}
-
+
# Source-specific or view-specific??
if indata.get('source'):
query['query']['bool']['must'].append({'term': {'sourceID': indata.get('source')}})
elif viewList:
query['query']['bool']['must'].append({'terms': {'sourceID': viewList}})
-
-
+
+
res = session.DB.ES.search(
index=session.DB.dbname,
doc_type="issue",
body = query
)
-
+
issue_contributors = []
for bucket in res['aggregations']['by_creator']['buckets']:
email = bucket['key']
if email not in issue_contributors:
issue_contributors.append(email)
-
+
for bucket in res['aggregations']['by_closer']['buckets']:
email = bucket['key']
if email not in issue_contributors:
issue_contributors.append(email)
-
+
# Now, for each contributor, find if they have done anything before
for email in issue_contributors:
ecr = find_earlier(session, query, dateFrom, email, 'created', 'issueCreator', 'issue', dOrg)
@@ -307,13 +307,13 @@ def run(API, environ, indata, session):
earliest = ecl
newcomers[email] = newcomers.get(email, {})
newcomers[email]['issue'] = earliest
-
+
email_contributors = []
-
+
################################
# For each newtimer, get a bio #
################################
-
+
for email in newcomers:
pid = hashlib.sha1( ("%s%s" % (dOrg, email)).encode('ascii', errors='replace')).hexdigest()
person = {}
@@ -321,11 +321,11 @@ def run(API, environ, indata, session):
person = session.DB.ES.get(index=session.DB.dbname, doc_type="person", id = pid)['_source']
person['md5'] = hashlib.md5(person['email'].encode('utf-8')).hexdigest() # gravatar needed for UI!
newcomers[email]['bio'] = person
-
+
newcomers_code = []
newcomers_issues = []
newcomers_email = []
-
+
# Count newcomers in each category (TODO: put this elsewhere earlier)
for email, entry in newcomers.items():
if 'code' in entry:
@@ -334,7 +334,7 @@ def run(API, environ, indata, session):
newcomers_issues.append(email)
if 'email' in entry:
newcomers_email.append(email)
-
+
JSON_OUT = {
'okay': True,
'stats': {
diff --git a/api/pages/bio/trends.py b/api/pages/bio/trends.py
index 776779e9..7e5e92b7 100644
--- a/api/pages/bio/trends.py
+++ b/api/pages/bio/trends.py
@@ -56,7 +56,7 @@
# security:
# - cookieAuth: []
# summary: Shows a quick trend summary of the past 6 months for a contributor
-#
+#
########################################################################
@@ -71,36 +71,36 @@
import time
def run(API, environ, indata, session):
-
+
# We need to be logged in for this!
if not session.user:
raise API.exception(403, "You must be logged in to use this API endpoint! %s")
-
+
now = time.time()
-
+
# First, fetch the view if we have such a thing enabled
viewList = []
if indata.get('view'):
viewList = session.getView(indata.get('view'))
if indata.get('subfilter'):
- viewList = session.subFilter(indata.get('subfilter'), view = viewList)
-
-
+ viewList = session.subFilter(indata.get('subfilter'), view = viewList)
+
+
dateTo = indata.get('to', int(time.time()))
dateFrom = indata.get('from', dateTo - (86400*30*6)) # Default to a 6 month span
if dateFrom < 0:
dateFrom = 0
dateYonder = dateFrom - (dateTo - dateFrom)
-
-
+
+
dOrg = session.user['defaultOrganisation'] or "apache"
-
+
####################################################################
# We start by doing all the queries for THIS period. #
# Then we reset the query, and change date to yonder-->from #
# and rerun the same queries. #
####################################################################
-
+
rangeKey = 'created'
rangeQuery = {'range':
{
@@ -139,8 +139,8 @@ def run(API, environ, indata, session):
{'term': {codeKey: indata.get('email')}},
]
query['query']['bool']['minimum_should_match'] = 1
-
-
+
+
# ISSUES CREATED
res = session.DB.ES.count(
index=session.DB.dbname,
@@ -148,8 +148,8 @@ def run(API, environ, indata, session):
body = query
)
no_issues_created = res['count']
-
-
+
+
# ISSUES CLOSED
rangeKey = "closed"
query['query']['bool']['must'][0] = {'range':
@@ -160,15 +160,15 @@ def run(API, environ, indata, session):
}
}
}
-
+
res = session.DB.ES.count(
index=session.DB.dbname,
doc_type="issue",
body = query
)
no_issues_closed = res['count']
-
-
+
+
# EMAIL SENT
rangeKey = "ts"
query['query']['bool']['must'][0] = {'range':
@@ -179,14 +179,14 @@ def run(API, environ, indata, session):
}
}
}
-
+
res = session.DB.ES.count(
index=session.DB.dbname,
doc_type="email",
body = query
)
no_email_sent = res['count']
-
+
# COMMITS MADE
rangeKey = "ts"
query['query']['bool']['must'][0] = {'range':
@@ -197,20 +197,20 @@ def run(API, environ, indata, session):
}
}
}
-
+
res = session.DB.ES.count(
index=session.DB.dbname,
doc_type="code_commit",
body = query
)
no_commits = res['count']
-
-
-
+
+
+
####################################################################
# Change to PRIOR SPAN #
####################################################################
-
+
# ISSUES OPENED
rangeKey = "created"
query['query']['bool']['must'][0] = {'range':
@@ -221,16 +221,16 @@ def run(API, environ, indata, session):
}
}
}
-
+
res = session.DB.ES.count(
index=session.DB.dbname,
doc_type="issue",
body = query
)
no_issues_created_before = res['count']
-
-
-
+
+
+
# ISSUES CLOSED
rangeKey = "closed"
query['query']['bool']['must'][0] = {'range':
@@ -241,15 +241,15 @@ def run(API, environ, indata, session):
}
}
}
-
+
res = session.DB.ES.count(
index=session.DB.dbname,
doc_type="issue",
body = query
)
no_issues_closed_before = res['count']
-
-
+
+
# EMAIL SENT
rangeKey = "ts"
query['query']['bool']['must'][0] = {'range':
@@ -260,15 +260,15 @@ def run(API, environ, indata, session):
}
}
}
-
-
+
+
res = session.DB.ES.count(
index=session.DB.dbname,
doc_type="email",
body = query
)
no_email_sent_before = res['count']
-
+
# CODE COMMITS
rangeKey = "ts"
query['query']['bool']['must'][0] = {'range':
@@ -279,16 +279,16 @@ def run(API, environ, indata, session):
}
}
}
-
-
+
+
res = session.DB.ES.count(
index=session.DB.dbname,
doc_type="code_commit",
body = query
)
no_commits_before = res['count']
-
-
+
+
trends = {
"created": {
'before': no_issues_created_before,
@@ -311,7 +311,7 @@ def run(API, environ, indata, session):
'title': "Commits this period"
}
}
-
+
JSON_OUT = {
'trends': trends,
'okay': True,
diff --git a/api/pages/ci/queue.py b/api/pages/ci/queue.py
index 2ef9f729..bba2f65d 100644
--- a/api/pages/ci/queue.py
+++ b/api/pages/ci/queue.py
@@ -56,7 +56,7 @@
# security:
# - cookieAuth: []
# summary: Shows CI queue over time
-#
+#
########################################################################
@@ -70,29 +70,29 @@
import hashlib
def run(API, environ, indata, session):
-
+
# We need to be logged in for this!
if not session.user:
raise API.exception(403, "You must be logged in to use this API endpoint! %s")
-
+
now = time.time()
-
+
# First, fetch the view if we have such a thing enabled
viewList = []
if indata.get('view'):
viewList = session.getView(indata.get('view'))
if indata.get('subfilter'):
- viewList = session.subFilter(indata.get('subfilter'), view = viewList)
-
+ viewList = session.subFilter(indata.get('subfilter'), view = viewList)
+
# We only want build sources, so we can sum up later.
viewList = session.subType(['jenkins', 'travis', 'buildbot'], viewList)
-
+
dateTo = indata.get('to', int(time.time()))
dateFrom = indata.get('from', dateTo - (86400*30*6)) # Default to a 6 month span
-
+
interval = indata.get('interval', 'month')
-
-
+
+
####################################################################
####################################################################
dOrg = session.user['defaultOrganisation'] or "apache"
@@ -120,13 +120,13 @@ def run(API, environ, indata, session):
# Source-specific or view-specific??
if indata.get('source'):
viewList = [indata.get('source')]
-
+
query['query']['bool']['must'].append({'term': {'sourceID': 'x'}})
-
+
timeseries = []
for source in viewList:
query['query']['bool']['must'][2] = {'term': {'sourceID': source}}
-
+
# Get queue stats
query['aggs'] = {
'timeseries': {
@@ -175,7 +175,7 @@ def run(API, environ, indata, session):
bucket['wait']['value'] = bucket['wait'].get('value', 0) or 0
if bucket['doc_count'] == 0:
continue
-
+
found = False
for t in timeseries:
if t['date'] == ts:
@@ -192,11 +192,11 @@ def run(API, environ, indata, session):
'average wait (hours)': bucket['wait']['value'],
'builders': 1,
})
-
+
for t in timeseries:
t['average wait (hours)'] = int(t['average wait (hours)']/360)/10.0
del t['builders']
-
+
JSON_OUT = {
'widgetType': {
'chartType': 'line', # Recommendation for the UI
diff --git a/api/pages/ci/status.py b/api/pages/ci/status.py
index a1a8aac8..2891a791 100644
--- a/api/pages/ci/status.py
+++ b/api/pages/ci/status.py
@@ -56,7 +56,7 @@
# security:
# - cookieAuth: []
# summary: Shows CI queue over time
-#
+#
########################################################################
@@ -70,27 +70,27 @@
import hashlib
def run(API, environ, indata, session):
-
+
# We need to be logged in for this!
if not session.user:
raise API.exception(403, "You must be logged in to use this API endpoint! %s")
-
+
now = time.time()
-
+
# First, fetch the view if we have such a thing enabled
viewList = []
if indata.get('view'):
viewList = session.getView(indata.get('view'))
if indata.get('subfilter'):
- viewList = session.subFilter(indata.get('subfilter'), view = viewList)
-
-
+ viewList = session.subFilter(indata.get('subfilter'), view = viewList)
+
+
dateTo = indata.get('to', int(time.time()))
dateFrom = indata.get('from', dateTo - (86400*30*6)) # Default to a 6 month span
-
+
interval = indata.get('interval', 'month')
-
-
+
+
####################################################################
####################################################################
dOrg = session.user['defaultOrganisation'] or "apache"
@@ -120,7 +120,7 @@ def run(API, environ, indata, session):
query['query']['bool']['must'].append({'term': {'sourceID': indata.get('source')}})
elif viewList:
query['query']['bool']['must'].append({'terms': {'sourceID': viewList}})
-
+
# Get queue stats
query['aggs'] = {
'timeseries': {
@@ -158,7 +158,7 @@ def run(API, environ, indata, session):
size = 0,
body = query
)
-
+
timeseries = []
for bucket in res['aggregations']['timeseries']['buckets']:
if bucket['doc_count'] == 0:
@@ -169,7 +169,7 @@ def run(API, environ, indata, session):
'builds blocked': bucket['blocked']['value'],
'builds stuck': bucket['stuck']['value']
})
-
+
JSON_OUT = {
'widgetType': {
'chartType': 'bar' # Recommendation for the UI
diff --git a/api/pages/ci/top-buildcount.py b/api/pages/ci/top-buildcount.py
index aa704058..d688346e 100644
--- a/api/pages/ci/top-buildcount.py
+++ b/api/pages/ci/top-buildcount.py
@@ -56,7 +56,7 @@
# security:
# - cookieAuth: []
# summary: Shows top 25 jobs by total builds done. Essentially buildtime, tweaked
-#
+#
########################################################################
@@ -72,23 +72,23 @@
import re
def run(API, environ, indata, session):
-
+
# We need to be logged in for this!
if not session.user:
raise API.exception(403, "You must be logged in to use this API endpoint! %s")
-
+
now = time.time()
-
+
# First, fetch the view if we have such a thing enabled
viewList = []
if indata.get('view'):
viewList = session.getView(indata.get('view'))
if indata.get('subfilter'):
- viewList = session.subFilter(indata.get('subfilter'), view = viewList)
-
+ viewList = session.subFilter(indata.get('subfilter'), view = viewList)
+
dateTo = indata.get('to', int(time.time()))
dateFrom = indata.get('from', dateTo - (86400*30*6)) # Default to a 6 month span
-
+
####################################################################
####################################################################
dOrg = session.user['defaultOrganisation'] or "apache"
@@ -118,7 +118,7 @@ def run(API, environ, indata, session):
query['query']['bool']['must'].append({'term': {'sourceID': indata.get('source')}})
elif viewList:
query['query']['bool']['must'].append({'terms': {'sourceID': viewList}})
-
+
query['aggs'] = {
'by_job': {
'terms': {
@@ -146,14 +146,14 @@ def run(API, environ, indata, session):
}
}
}
-
+
res = session.DB.ES.search(
index=session.DB.dbname,
doc_type="ci_build",
size = 0,
body = query
)
-
+
jobs = []
for doc in res['aggregations']['by_job']['buckets']:
job = doc['key']
@@ -162,12 +162,12 @@ def run(API, environ, indata, session):
ci = doc['ci']['buckets'][0]['key']
jobname = doc['name']['buckets'][0]['key']
jobs.append([builds, duration, jobname, ci])
-
+
topjobs = sorted(jobs, key = lambda x: int(x[0]), reverse = True)
tophash = {}
for v in topjobs:
tophash["%s (%s)" % (v[2], v[3])] = v[0]
-
+
JSON_OUT = {
'counts': tophash,
'okay': True,
diff --git a/api/pages/ci/top-buildtime.py b/api/pages/ci/top-buildtime.py
index 6aded754..46fafca9 100644
--- a/api/pages/ci/top-buildtime.py
+++ b/api/pages/ci/top-buildtime.py
@@ -56,7 +56,7 @@
# security:
# - cookieAuth: []
# summary: Shows top 25 jobs by total build time spent
-#
+#
########################################################################
@@ -72,23 +72,23 @@
import re
def run(API, environ, indata, session):
-
+
# We need to be logged in for this!
if not session.user:
raise API.exception(403, "You must be logged in to use this API endpoint! %s")
-
+
now = time.time()
-
+
# First, fetch the view if we have such a thing enabled
viewList = []
if indata.get('view'):
viewList = session.getView(indata.get('view'))
if indata.get('subfilter'):
- viewList = session.subFilter(indata.get('subfilter'), view = viewList)
-
+ viewList = session.subFilter(indata.get('subfilter'), view = viewList)
+
dateTo = indata.get('to', int(time.time()))
dateFrom = indata.get('from', dateTo - (86400*30*6)) # Default to a 6 month span
-
+
####################################################################
####################################################################
dOrg = session.user['defaultOrganisation'] or "apache"
@@ -118,7 +118,7 @@ def run(API, environ, indata, session):
query['query']['bool']['must'].append({'term': {'sourceID': indata.get('source')}})
elif viewList:
query['query']['bool']['must'].append({'terms': {'sourceID': viewList}})
-
+
query['aggs'] = {
'by_job': {
'terms': {
@@ -146,14 +146,14 @@ def run(API, environ, indata, session):
}
}
}
-
+
res = session.DB.ES.search(
index=session.DB.dbname,
doc_type="ci_build",
size = 0,
body = query
)
-
+
jobs = []
for doc in res['aggregations']['by_job']['buckets']:
job = doc['key']
@@ -162,7 +162,7 @@ def run(API, environ, indata, session):
ci = doc['ci']['buckets'][0]['key']
jobname = doc['name']['buckets'][0]['key']
jobs.append([builds, duration, jobname, ci])
-
+
topjobs = sorted(jobs, key = lambda x: int(x[1]), reverse = True)
top = topjobs[0:24]
if len(topjobs) > 25:
@@ -170,11 +170,11 @@ def run(API, environ, indata, session):
for repo in topjobs[24:]:
count += repo[1]
top.append([1, count, "Other jobs", '??'])
-
+
tophash = {}
for v in top:
tophash["%s (%s)" % (v[2], v[3])] = int((v[1]/360000))/10
-
+
JSON_OUT = {
'counts': tophash,
'okay': True,
diff --git a/api/pages/code/changes.py b/api/pages/code/changes.py
index c6233d4f..72a90cf1 100644
--- a/api/pages/code/changes.py
+++ b/api/pages/code/changes.py
@@ -56,7 +56,7 @@
# security:
# - cookieAuth: []
# summary: Show insertions/deletions as a timeseries
-#
+#
########################################################################
@@ -71,33 +71,33 @@
import time
def run(API, environ, indata, session):
-
+
# We need to be logged in for this!
if not session.user:
raise API.exception(403, "You must be logged in to use this API endpoint! %s")
-
+
now = time.time()
-
+
# First, fetch the view if we have such a thing enabled
viewList = []
if indata.get('view'):
viewList = session.getView(indata.get('view'))
if indata.get('subfilter'):
- viewList = session.subFilter(indata.get('subfilter'), view = viewList)
-
-
+ viewList = session.subFilter(indata.get('subfilter'), view = viewList)
+
+
dateTo = indata.get('to', int(time.time()))
dateFrom = indata.get('from', dateTo - (86400*30*6)) # Default to a 6 month span
-
+
which = 'committer_email'
role = 'committer'
if indata.get('author', False):
which = 'author_email'
role = 'author'
-
+
interval = indata.get('interval', 'day')
-
-
+
+
####################################################################
####################################################################
dOrg = session.user['defaultOrganisation'] or "apache"
@@ -130,7 +130,7 @@ def run(API, environ, indata, session):
if indata.get('email'):
query['query']['bool']['should'] = [{'term': {'committer_email': indata.get('email')}}, {'term': {'author_email': indata.get('email')}}]
query['query']['bool']['minimum_should_match'] = 1
-
+
# Path filter?
if indata.get('pathfilter'):
pf = indata.get('pathfilter')
@@ -140,7 +140,7 @@ def run(API, environ, indata, session):
query['query']['bool']['must_not'].append({'regexp': {'files_changed': pf}})
else:
query['query']['bool']['must'].append({'regexp': {'files_changed': pf}})
-
+
# Get timeseries for this period
query['aggs'] = {
'per_interval': {
@@ -162,7 +162,7 @@ def run(API, environ, indata, session):
}
}
}
-
+
res = session.DB.ES.search(
index=session.DB.dbname,
doc_type="code_commit",
@@ -180,7 +180,7 @@ def run(API, environ, indata, session):
'insertions': icount,
'deletions': dcount
})
-
+
JSON_OUT = {
'timeseries': timeseries,
'interval': interval,
diff --git a/api/pages/code/commits.py b/api/pages/code/commits.py
index 2899f756..54bb4763 100644
--- a/api/pages/code/commits.py
+++ b/api/pages/code/commits.py
@@ -56,7 +56,7 @@
# security:
# - cookieAuth: []
# summary: Show commits as a timeseries
-#
+#
########################################################################
@@ -72,33 +72,33 @@
import hashlib
def run(API, environ, indata, session):
-
+
# We need to be logged in for this!
if not session.user:
raise API.exception(403, "You must be logged in to use this API endpoint! %s")
-
+
now = time.time()
-
+
# First, fetch the view if we have such a thing enabled
viewList = []
if indata.get('view'):
viewList = session.getView(indata.get('view'))
if indata.get('subfilter'):
- viewList = session.subFilter(indata.get('subfilter'), view = viewList)
-
-
+ viewList = session.subFilter(indata.get('subfilter'), view = viewList)
+
+
dateTo = indata.get('to', int(time.time()))
dateFrom = indata.get('from', dateTo - (86400*30*6)) # Default to a 6 month span
-
+
which = 'committer_email'
role = 'committer'
if indata.get('author', False):
which = 'author_email'
role = 'author'
-
+
interval = indata.get('interval', 'day')
-
-
+
+
####################################################################
####################################################################
dOrg = session.user['defaultOrganisation'] or "apache"
@@ -131,7 +131,7 @@ def run(API, environ, indata, session):
if indata.get('email'):
query['query']['bool']['should'] = [{'term': {'committer_email': indata.get('email')}}, {'term': {'author_email': indata.get('email')}}]
query['query']['bool']['minimum_should_match'] = 1
-
+
# Path filter?
if indata.get('pathfilter'):
pf = indata.get('pathfilter')
@@ -141,14 +141,14 @@ def run(API, environ, indata, session):
query['query']['bool']['must_not'].append({'regexp': {'files_changed': pf}})
else:
query['query']['bool']['must'].append({'regexp': {'files_changed': pf}})
-
+
# Get number of committers, this period
query['aggs'] = {
'commits': {
'date_histogram': {
'field': 'date',
'interval': interval
- }
+ }
}
}
res = session.DB.ES.search(
@@ -157,7 +157,7 @@ def run(API, environ, indata, session):
size = 0,
body = query
)
-
+
timeseries = []
for bucket in res['aggregations']['commits']['buckets']:
ts = int(bucket['key'] / 1000)
@@ -166,7 +166,7 @@ def run(API, environ, indata, session):
'date': ts,
'commits': count
})
-
+
JSON_OUT = {
'widgetType': {
'chartType': 'bar' # Recommendation for the UI
diff --git a/api/pages/code/committers.py b/api/pages/code/committers.py
index 7b6d5183..a1370984 100644
--- a/api/pages/code/committers.py
+++ b/api/pages/code/committers.py
@@ -56,7 +56,7 @@
# security:
# - cookieAuth: []
# summary: Shows trend data for a set of repos over a given period of time
-#
+#
########################################################################
@@ -72,33 +72,33 @@
import hashlib
def run(API, environ, indata, session):
-
+
# We need to be logged in for this!
if not session.user:
raise API.exception(403, "You must be logged in to use this API endpoint! %s")
-
+
now = time.time()
-
+
# First, fetch the view if we have such a thing enabled
viewList = []
if indata.get('view'):
viewList = session.getView(indata.get('view'))
if indata.get('subfilter'):
- viewList = session.subFilter(indata.get('subfilter'), view = viewList)
-
-
+ viewList = session.subFilter(indata.get('subfilter'), view = viewList)
+
+
dateTo = indata.get('to', int(time.time()))
dateFrom = indata.get('from', dateTo - (86400*30*6)) # Default to a 6 month span
-
+
which = 'committer_email'
role = 'committer'
if indata.get('author', False):
which = 'author_email'
role = 'author'
-
+
interval = indata.get('interval', 'month')
-
-
+
+
####################################################################
####################################################################
dOrg = session.user['defaultOrganisation'] or "apache"
@@ -131,7 +131,7 @@ def run(API, environ, indata, session):
if indata.get('email'):
query['query']['bool']['should'] = [{'term': {'committer_email': indata.get('email')}}, {'term': {'author_email': indata.get('email')}}]
query['query']['bool']['minimum_should_match'] = 1
-
+
# Path filter?
if indata.get('pathfilter'):
pf = indata.get('pathfilter')
@@ -141,7 +141,7 @@ def run(API, environ, indata, session):
query['query']['bool']['must_not'].append({'regexp': {'files_changed': pf}})
else:
query['query']['bool']['must'].append({'regexp': {'files_changed': pf}})
-
+
# Get top 25 committers this period
query['aggs'] = {
'committers': {
@@ -176,7 +176,7 @@ def run(API, environ, indata, session):
},
}
},
-
+
}
res = session.DB.ES.search(
index=session.DB.dbname,
@@ -205,12 +205,12 @@ def run(API, environ, indata, session):
'insertions': int(bucket['byinsertions']['buckets'][0]['stats']['value']),
'deletions': int(bucket['bydeletions']['buckets'][0]['stats']['value'])
}
-
+
topN = []
for email, person in people.items():
topN.append(person)
topN = sorted(topN, key = lambda x: x['count'], reverse = True)
-
+
# Get timeseries for this period
query['aggs'] = {
'per_interval': {
@@ -232,7 +232,7 @@ def run(API, environ, indata, session):
}
}
}
-
+
res = session.DB.ES.search(
index=session.DB.dbname,
doc_type="code_commit",
@@ -250,7 +250,7 @@ def run(API, environ, indata, session):
'committers': ccount,
'authors': acount
})
-
+
JSON_OUT = {
'topN': {
'denoter': 'commits',
diff --git a/api/pages/code/evolution.py b/api/pages/code/evolution.py
index 593bd47c..8bc159bc 100644
--- a/api/pages/code/evolution.py
+++ b/api/pages/code/evolution.py
@@ -56,7 +56,7 @@
# security:
# - cookieAuth: []
# summary: Show code evolution as a timeseries
-#
+#
########################################################################
@@ -72,25 +72,25 @@
import hashlib
def run(API, environ, indata, session):
-
+
# We need to be logged in for this!
if not session.user:
raise API.exception(403, "You must be logged in to use this API endpoint! %s")
-
+
now = time.time()
-
+
# First, fetch the view if we have such a thing enabled
viewList = []
if indata.get('view'):
viewList = session.getView(indata.get('view'))
if indata.get('subfilter'):
- viewList = session.subFilter(indata.get('subfilter'), view = viewList)
-
-
+ viewList = session.subFilter(indata.get('subfilter'), view = viewList)
+
+
breakdown = False
onlycode = False
-
-
+
+
####################################################################
####################################################################
dOrg = session.user['defaultOrganisation'] or "apache"
@@ -120,7 +120,7 @@ def run(API, environ, indata, session):
query['query']['bool']['must'].append({'term': {'sourceID': indata.get('source')}})
elif viewList:
query['query']['bool']['must'].append({'terms': {'sourceID': viewList}})
-
+
# We need scrolling here!
res = session.DB.ES.search(
index=session.DB.dbname,
@@ -133,10 +133,10 @@ def run(API, environ, indata, session):
scroll_size = res['hits']['total']
if type(scroll_size) is dict:
scroll_size = scroll_size['value'] # ES >= 7.x
-
+
timeseries = []
tstmp = {}
-
+
while (scroll_size > 0):
for doc in res['hits']['hits']:
updates = doc['_source']
@@ -151,15 +151,15 @@ def run(API, environ, indata, session):
item['code'] = item.get('code', 0) + (updates['loc'] or 0)
item['comments'] = item.get('comments', 0) + (updates['comments'] or 0)
item['blanks'] = item.get('blanks', 0) + (updates['blank'] or 0)
-
+
res = session.DB.ES.scroll(scroll_id = sid, scroll = '1m')
sid = res['_scroll_id']
scroll_size = len(res['hits']['hits'])
-
+
for k, v in tstmp.items():
v['date'] = k
timeseries.append(v)
-
+
timeseries = sorted(timeseries, key = lambda x: x['date'])
JSON_OUT = {
'widgetType': {
diff --git a/api/pages/code/pony-timeseries.py b/api/pages/code/pony-timeseries.py
index a7c1c8ec..5427b440 100644
--- a/api/pages/code/pony-timeseries.py
+++ b/api/pages/code/pony-timeseries.py
@@ -56,7 +56,7 @@
# security:
# - cookieAuth: []
# summary: Shows timeseries of Pony Factor over time
-#
+#
########################################################################
@@ -74,31 +74,31 @@
import dateutil.relativedelta
def run(API, environ, indata, session):
-
+
# We need to be logged in for this!
if not session.user:
raise API.exception(403, "You must be logged in to use this API endpoint! %s")
-
+
now = time.time()
-
+
# First, fetch the view if we have such a thing enabled
viewList = []
if indata.get('view'):
viewList = session.getView(indata.get('view'))
if indata.get('subfilter'):
- viewList = session.subFilter(indata.get('subfilter'), view = viewList)
-
-
+ viewList = session.subFilter(indata.get('subfilter'), view = viewList)
+
+
hl = indata.get('span', 24)
tnow = datetime.date.today()
nm = tnow.month - (tnow.month % 3)
ny = tnow.year
ts = []
-
+
if nm < 1:
nm += 12
ny = ny - 1
-
+
while ny > 1970:
d = datetime.date(ny, nm, 1)
t = time.mktime(d.timetuple())
@@ -108,8 +108,8 @@ def run(API, environ, indata, session):
if nm < 1:
nm += 12
ny = ny - 1
-
-
+
+
####################################################################
####################################################################
dOrg = session.user['defaultOrganisation'] or "apache"
@@ -139,31 +139,31 @@ def run(API, environ, indata, session):
query['query']['bool']['must'].append({'term': {'sourceID': indata.get('source')}})
elif viewList:
query['query']['bool']['must'].append({'terms': {'sourceID': viewList}})
-
+
# Get an initial count of commits
res = session.DB.ES.count(
index=session.DB.dbname,
doc_type="code_commit",
body = query
)
-
+
globcount = res['count']
if globcount == 0:
break
-
+
# Get top 25 committers this period
query['aggs'] = {
'by_committer': {
'terms': {
'field': 'committer_email',
'size': 1000
- }
+ }
},
'by_author': {
'terms': {
'field': 'author_email',
'size': 1000
- }
+ }
}
}
res = session.DB.ES.search(
@@ -172,8 +172,8 @@ def run(API, environ, indata, session):
size = 0,
body = query
)
-
-
+
+
# PF for committers
pf_committer = 0
pf_committer_count = 0
@@ -183,7 +183,7 @@ def run(API, environ, indata, session):
pf_committer_count += count
if pf_committer_count > int(globcount/2):
break
-
+
# PF for authors
pf_author = 0
pf_author_count = 0
@@ -203,9 +203,9 @@ def run(API, environ, indata, session):
'Pony Factor (authorship)': pf_author,
'Meta-Pony Factor': len(cpf)
})
-
+
ts = sorted(ts, key = lambda x: x['date'])
-
+
JSON_OUT = {
'text': "This shows Pony Factors as calculated over a %u month timespan. Authorship measures the people writing the bulk of the codebase, committership mesaures the people committing (merging) the code, and meta-pony is an estimation of how many organisations/companies are involved." % hl,
'timeseries': ts,
diff --git a/api/pages/code/pony.py b/api/pages/code/pony.py
index 3eb074fe..ee64c8b5 100644
--- a/api/pages/code/pony.py
+++ b/api/pages/code/pony.py
@@ -56,7 +56,7 @@
# security:
# - cookieAuth: []
# summary: Shows pony factor data for a set of repos over a given period of time
-#
+#
########################################################################
@@ -72,28 +72,28 @@
import re
def run(API, environ, indata, session):
-
+
# We need to be logged in for this!
if not session.user:
raise API.exception(403, "You must be logged in to use this API endpoint! %s")
-
+
now = time.time()
-
+
# First, fetch the view if we have such a thing enabled
viewList = []
if indata.get('view'):
viewList = session.getView(indata.get('view'))
if indata.get('subfilter'):
- viewList = session.subFilter(indata.get('subfilter'), view = viewList)
-
-
+ viewList = session.subFilter(indata.get('subfilter'), view = viewList)
+
+
dateTo = indata.get('to', int(time.time()))
dateFrom = indata.get('from', dateTo - (86400*30*24)) # Default to a 24 month span
if dateFrom < 0:
dateFrom = 0
dateYonder = dateFrom - (dateTo - dateFrom)
-
-
+
+
####################################################################
####################################################################
dOrg = session.user['defaultOrganisation'] or "apache"
@@ -123,29 +123,29 @@ def run(API, environ, indata, session):
query['query']['bool']['must'].append({'term': {'sourceID': indata.get('source')}})
elif viewList:
query['query']['bool']['must'].append({'terms': {'sourceID': viewList}})
-
+
# Get an initial count of commits
res = session.DB.ES.count(
index=session.DB.dbname,
doc_type="code_commit",
body = query
)
-
+
globcount = res['count']
-
+
# Get top 25 committers this period
query['aggs'] = {
'by_committer': {
'terms': {
'field': 'committer_email',
'size': 5000
- }
+ }
},
'by_author': {
'terms': {
'field': 'author_email',
'size': 5000
- }
+ }
}
}
res = session.DB.ES.search(
@@ -154,8 +154,8 @@ def run(API, environ, indata, session):
size = 0,
body = query
)
-
-
+
+
# PF for committers
pf_committer = 0
pf_committer_count = 0
@@ -165,7 +165,7 @@ def run(API, environ, indata, session):
pf_committer_count += count
if pf_committer_count > int(globcount/2):
break
-
+
# PF for authors
pf_author = 0
pf_author_count = 0
@@ -178,8 +178,8 @@ def run(API, environ, indata, session):
cpf[mldom] = True
if pf_author_count > int(globcount/2):
break
-
-
+
+
####################################################################
####################################################################
dOrg = session.user['defaultOrganisation'] or "apache"
@@ -209,29 +209,29 @@ def run(API, environ, indata, session):
query['query']['bool']['must'].append({'term': {'sourceID': indata.get('source')}})
elif viewList:
query['query']['bool']['must'].append({'terms': {'sourceID': viewList}})
-
+
# Get an initial count of commits
res = session.DB.ES.count(
index=session.DB.dbname,
doc_type="code_commit",
body = query
)
-
+
globcount = res['count']
-
+
# Get top 25 committers this period
query['aggs'] = {
'by_committer': {
'terms': {
'field': 'committer_email',
'size': 5000
- }
+ }
},
'by_author': {
'terms': {
'field': 'author_email',
'size': 5000
- }
+ }
}
}
res = session.DB.ES.search(
@@ -240,8 +240,8 @@ def run(API, environ, indata, session):
size = 0,
body = query
)
-
-
+
+
# PF for committers
pf_committer_b = 0
pf_committer_count = 0
@@ -251,7 +251,7 @@ def run(API, environ, indata, session):
pf_committer_count += count
if pf_committer_count > int(globcount/2):
break
-
+
# PF for authors
pf_author_b = 0
pf_author_count = 0
@@ -264,7 +264,7 @@ def run(API, environ, indata, session):
cpf_b[mldom] = True
if pf_author_count > int(globcount/2):
break
-
+
JSON_OUT = {
'factors': [
{
diff --git a/api/pages/code/punchcard.py b/api/pages/code/punchcard.py
index ab0a52f8..a8bf26bd 100644
--- a/api/pages/code/punchcard.py
+++ b/api/pages/code/punchcard.py
@@ -56,7 +56,7 @@
# security:
# - cookieAuth: []
# summary: Show commits as a timeseries
-#
+#
########################################################################
@@ -72,33 +72,33 @@
import hashlib
def run(API, environ, indata, session):
-
+
# We need to be logged in for this!
if not session.user:
raise API.exception(403, "You must be logged in to use this API endpoint! %s")
-
+
now = time.time()
-
+
# First, fetch the view if we have such a thing enabled
viewList = []
if indata.get('view'):
viewList = session.getView(indata.get('view'))
if indata.get('subfilter'):
- viewList = session.subFilter(indata.get('subfilter'), view = viewList)
-
-
+ viewList = session.subFilter(indata.get('subfilter'), view = viewList)
+
+
dateTo = indata.get('to', int(time.time()))
dateFrom = indata.get('from', dateTo - (86400*30*6)) # Default to a 6 month span
-
+
which = 'committer_email'
role = 'committer'
if indata.get('author', False):
which = 'author_email'
role = 'author'
-
+
interval = indata.get('interval', 'day')
-
-
+
+
####################################################################
####################################################################
dOrg = session.user['defaultOrganisation'] or "apache"
@@ -131,7 +131,7 @@ def run(API, environ, indata, session):
if indata.get('email'):
query['query']['bool']['should'] = [{'term': {'committer_email': indata.get('email')}}, {'term': {'author_email': indata.get('email')}}]
query['query']['bool']['minimum_should_match'] = 1
-
+
# Path filter?
if indata.get('pathfilter'):
pf = indata.get('pathfilter')
@@ -141,7 +141,7 @@ def run(API, environ, indata, session):
query['query']['bool']['must_not'].append({'regexp': {'files_changed': pf}})
else:
query['query']['bool']['must'].append({'regexp': {'files_changed': pf}})
-
+
# Get number of committers, this period
query['aggs'] = {
'commits': {
@@ -149,7 +149,7 @@ def run(API, environ, indata, session):
'field': 'date',
'interval': 'hour',
"format": "E - k"
- }
+ }
}
}
res = session.DB.ES.search(
@@ -158,7 +158,7 @@ def run(API, environ, indata, session):
size = 0,
body = query
)
-
+
timeseries = {}
for bucket in res['aggregations']['commits']['buckets']:
ts = bucket['key_as_string']
diff --git a/api/pages/code/relationships.py b/api/pages/code/relationships.py
index 843c1da6..1786b7e6 100644
--- a/api/pages/code/relationships.py
+++ b/api/pages/code/relationships.py
@@ -56,7 +56,7 @@
# security:
# - cookieAuth: []
# summary: Shows a breakdown of contributor relationships between repositories
-#
+#
########################################################################
@@ -75,32 +75,32 @@
import math
def run(API, environ, indata, session):
-
+
# We need to be logged in for this!
if not session.user:
raise API.exception(403, "You must be logged in to use this API endpoint! %s")
-
+
now = time.time()
-
+
# First, fetch the view if we have such a thing enabled
viewList = []
if indata.get('view'):
viewList = session.getView(indata.get('view'))
if indata.get('subfilter'):
- viewList = session.subFilter(indata.get('subfilter'), view = viewList)
-
+ viewList = session.subFilter(indata.get('subfilter'), view = viewList)
+
dateTo = indata.get('to', int(time.time()))
dateFrom = indata.get('from', dateTo - (86400*30*6)) # Default to a 6 month span
-
+
which = 'committer_email'
role = 'committer'
if indata.get('author', False):
which = 'author_email'
role = 'author'
-
+
interval = indata.get('interval', 'day')
-
-
+
+
####################################################################
####################################################################
dOrg = session.user['defaultOrganisation'] or "apache"
@@ -132,14 +132,14 @@ def run(API, environ, indata, session):
query['query']['bool']['must'].append({'terms': {'sourceID': viewList}})
if indata.get('email'):
query['query']['bool']['must'].append({'term': {'committer_email' if not indata.get('author') else 'author_email': indata.get('email')}})
-
+
# Get number of commits, this period, per repo
query['aggs'] = {
'per_repo': {
'terms': {
'field': 'sourceID',
'size': 10000
- }
+ }
}
}
res = session.DB.ES.search(
@@ -148,7 +148,7 @@ def run(API, environ, indata, session):
size = 0,
body = query
)
-
+
repos = {}
repo_commits = {}
authorlinks = {}
@@ -157,19 +157,19 @@ def run(API, environ, indata, session):
max_shared = 0
max_authors = 0
minLinks = indata.get('links', 1)
-
+
# For each repo, count commits and gather data on authors
for doc in res['aggregations']['per_repo']['buckets']:
sourceID = doc['key']
commits = doc['doc_count']
-
+
# Gather the unique authors/committers
query['aggs'] = {
'per_contributor': {
'terms': {
'field': 'committer_email' if not indata.get('author') else 'author_email',
'size': 10000
- }
+ }
}
}
xquery = copy.deepcopy(query)
@@ -187,7 +187,7 @@ def run(API, environ, indata, session):
max_commits = commits
repos[sourceID] = authors
repo_commits[sourceID] = commits
-
+
# Now, figure out which repos share the same contributors
repo_links = {}
repo_notoriety = {}
@@ -200,7 +200,7 @@ def run(API, environ, indata, session):
if not session.DB.ES.exists(index=session.DB.dbname, doc_type="source", id = ID):
continue
repodatas[ID] = session.DB.ES.get(index=session.DB.dbname, doc_type="source", id = ID)
-
+
for ID, repo in repos.items():
mylinks = {}
if not ID in repodatas:
@@ -237,11 +237,11 @@ def run(API, environ, indata, session):
if ID not in repo_notoriety:
repo_notoriety[ID] = set()
repo_notoriety[ID].update(mylinks.keys()) # How many projects is this repo connected to?
-
+
if ID not in repo_authors:
repo_authors[ID] = set()
repo_authors[ID].update(repo) # How many projects is this repo connected to?
-
+
if ID != oID:
repo_commits[ID] = repo_commits.get(ID, 0) + repo_commits[oID]
if repo_commits[ID] > max_commits:
@@ -250,7 +250,7 @@ def run(API, environ, indata, session):
max_links = len(repo_notoriety[ID])
if len(repo_authors[ID]) > max_authors:
max_authors = len(repo_authors[ID]) # Used for calculating max sphere size in charts
-
+
# Now, pull it all together!
nodes = []
links = []
@@ -273,7 +273,7 @@ def run(API, environ, indata, session):
}
nodes.append(doc)
existing_repos.append(sourceID)
-
+
for k, s in repo_links.items():
size = s
fr, to = k.split('@')
@@ -286,7 +286,7 @@ def run(API, environ, indata, session):
'tooltip': "%u committers in common" % size
}
links.append(doc)
-
+
JSON_OUT = {
'maxLinks': max_links,
'maxShared': max_shared,
diff --git a/api/pages/code/retention.py b/api/pages/code/retention.py
index 6e108441..70a7bc7b 100644
--- a/api/pages/code/retention.py
+++ b/api/pages/code/retention.py
@@ -56,7 +56,7 @@
# security:
# - cookieAuth: []
# summary: Shows retention metrics for a set of repos over a given period of time
-#
+#
########################################################################
@@ -73,37 +73,37 @@
import datetime
def run(API, environ, indata, session):
-
+
# We need to be logged in for this!
if not session.user:
raise API.exception(403, "You must be logged in to use this API endpoint! %s")
-
+
now = time.time()
-
+
# First, fetch the view if we have such a thing enabled
viewList = []
if indata.get('view'):
viewList = session.getView(indata.get('view'))
if indata.get('subfilter'):
- viewList = session.subFilter(indata.get('subfilter'), view = viewList)
-
-
+ viewList = session.subFilter(indata.get('subfilter'), view = viewList)
+
+
hl = indata.get('span', 12) # By default, we define a contributor as active if having committer in the past year
tnow = datetime.date.today()
nm = tnow.month - (tnow.month % 3)
ny = tnow.year
cy = ny
ts = []
-
+
if nm < 1:
nm += 12
ny = ny - 1
-
+
peopleSeen = {}
activePeople = {}
allPeople = {}
FoundSomething = False
-
+
ny = 1970
while ny < cy or (ny == cy and (nm+3) <= tnow.month):
d = datetime.date(ny, nm, 1)
@@ -116,7 +116,7 @@ def run(API, environ, indata, session):
break
d = datetime.date(ny, nm, 1)
tf = time.mktime(d.timetuple())
-
+
####################################################################
####################################################################
dOrg = session.user['defaultOrganisation'] or "apache"
@@ -146,32 +146,32 @@ def run(API, environ, indata, session):
query['query']['bool']['must'].append({'term': {'sourceID': indata.get('source')}})
elif viewList:
query['query']['bool']['must'].append({'terms': {'sourceID': viewList}})
-
+
# Get an initial count of commits
res = session.DB.ES.count(
index=session.DB.dbname,
doc_type="code_commit",
body = query
)
-
+
globcount = res['count']
if globcount == 0 and not FoundSomething:
continue
FoundSomething = True
-
+
# Get top 1000 committers this period
query['aggs'] = {
'by_committer': {
'terms': {
'field': 'committer_email',
'size': 25000
- }
+ }
},
'by_author': {
'terms': {
'field': 'author_email',
'size': 25000
- }
+ }
}
}
res = session.DB.ES.search(
@@ -180,12 +180,12 @@ def run(API, environ, indata, session):
size = 0,
body = query
)
-
-
+
+
retained = 0
added = 0
lost = 0
-
+
thisPeriod = []
for bucket in res['aggregations']['by_author']['buckets']:
who = bucket['key']
@@ -196,18 +196,18 @@ def run(API, environ, indata, session):
activePeople[who] = tf
if who not in allPeople:
allPeople[who] = tf
-
+
prune = []
for k, v in activePeople.items():
if v < (t - (hl*30.45*86400)):
prune.append(k)
lost += 1
-
+
for who in prune:
del activePeople[who]
del peopleSeen[who]
retained = len(activePeople) - added
-
+
ts.append({
'date': tf,
'People who (re)joined': added,
@@ -215,14 +215,14 @@ def run(API, environ, indata, session):
'People retained': retained,
'Active people': added + retained
})
-
+
groups = [
['More than 5 years', (5*365*86400)+1],
['2 - 5 years', (2*365*86400)+1],
['1 - 2 years', (365*86400)],
['Less than a year', 1]
]
-
+
counts = {}
totExp = 0
for person, age in activePeople.items():
@@ -232,7 +232,7 @@ def run(API, environ, indata, session):
counts[el[0]] = counts.get(el[0], 0) + 1
break
avgyr = (totExp / (86400*365)) / max(len(activePeople),1)
-
+
ts = sorted(ts, key = lambda x: x['date'])
avgm = ""
yr = int(avgyr)
diff --git a/api/pages/code/sloc.py b/api/pages/code/sloc.py
index 29b54c55..a6d7fc22 100644
--- a/api/pages/code/sloc.py
+++ b/api/pages/code/sloc.py
@@ -56,7 +56,7 @@
# security:
# - cookieAuth: []
# summary: Shows a breakdown of lines of code for one or more sources
-#
+#
########################################################################
@@ -70,20 +70,20 @@
import json
def run(API, environ, indata, session):
-
+
# We need to be logged in for this!
if not session.user:
raise API.exception(403, "You must be logged in to use this API endpoint! %s")
-
-
+
+
# First, fetch the view if we have such a thing enabled
viewList = []
if indata.get('view'):
viewList = session.getView(indata.get('view'))
if indata.get('subfilter'):
- viewList = session.subFilter(indata.get('subfilter'), view = viewList)
-
-
+ viewList = session.subFilter(indata.get('subfilter'), view = viewList)
+
+
# Fetch all sources for default org
dOrg = session.user['defaultOrganisation'] or "apache"
query = {
@@ -109,14 +109,14 @@ def run(API, environ, indata, session):
query['query']['bool']['must'].append({'term': {'sourceID': indata.get('source')}})
elif viewList:
query['query']['bool']['must'].append({'terms': {'sourceID': viewList}})
-
+
res = session.DB.ES.search(
index=session.DB.dbname,
doc_type="source",
size = 5000,
body = query
)
-
+
languages = {}
years = 0
for hit in res['hits']['hits']:
@@ -129,8 +129,8 @@ def run(API, environ, indata, session):
languages[k]['code'] += v.get('code', 0)
languages[k]['comment'] += v.get('comment', 0)
languages[k]['blank'] += v.get('blank', 0)
-
-
+
+
JSON_OUT = {
'languages': languages,
'okay': True,
diff --git a/api/pages/code/top-commits.py b/api/pages/code/top-commits.py
index ce75268e..d811082b 100644
--- a/api/pages/code/top-commits.py
+++ b/api/pages/code/top-commits.py
@@ -56,7 +56,7 @@
# security:
# - cookieAuth: []
# summary: Shows top 25 repos by commit volume
-#
+#
########################################################################
@@ -72,24 +72,24 @@
import re
def run(API, environ, indata, session):
-
+
# We need to be logged in for this!
if not session.user:
raise API.exception(403, "You must be logged in to use this API endpoint! %s")
-
+
now = time.time()
-
+
# First, fetch the view if we have such a thing enabled
viewList = []
if indata.get('view'):
viewList = session.getView(indata.get('view'))
if indata.get('subfilter'):
- viewList = session.subFilter(indata.get('subfilter'), view = viewList)
-
-
+ viewList = session.subFilter(indata.get('subfilter'), view = viewList)
+
+
dateTo = indata.get('to', int(time.time()))
dateFrom = indata.get('from', dateTo - (86400*30*6)) # Default to a 6 month span
-
+
####################################################################
####################################################################
dOrg = session.user['defaultOrganisation'] or "apache"
@@ -122,7 +122,7 @@ def run(API, environ, indata, session):
if indata.get('email'):
query['query']['bool']['should'] = [{'term': {'committer_email': indata.get('email')}}, {'term': {'author_email': indata.get('email')}}]
query['query']['bool']['minimum_should_match'] = 1
-
+
# Path filter?
if indata.get('pathfilter'):
pf = indata.get('pathfilter')
@@ -132,15 +132,15 @@ def run(API, environ, indata, session):
query['query']['bool']['must_not'].append({'regexp': {'files_changed': pf}})
else:
query['query']['bool']['must'].append({'regexp': {'files_changed': pf}})
-
-
+
+
# Get top 25 committers this period
query['aggs'] = {
'by_repo': {
'terms': {
'field': 'sourceURL',
'size': 5000
- }
+ }
}
}
res = session.DB.ES.search(
@@ -149,14 +149,14 @@ def run(API, environ, indata, session):
size = 0,
body = query
)
-
+
toprepos = []
for bucket in res['aggregations']['by_repo']['buckets']:
repo = re.sub(r".+/([^/]+?)(?:\.git)?$", r"\1", bucket['key'])
count = bucket['doc_count']
-
+
toprepos.append([repo, count])
-
+
toprepos = sorted(toprepos, key = lambda x: x[1], reverse = True)
top = toprepos[0:24]
if len(toprepos) > 25:
@@ -164,11 +164,11 @@ def run(API, environ, indata, session):
for repo in toprepos[25:]:
count += repo[1]
top.append(["Other repos", count])
-
+
tophash = {}
for v in top:
tophash[v[0]] = v[1]
-
+
JSON_OUT = {
'counts': tophash,
'okay': True,
diff --git a/api/pages/code/top-sloc.py b/api/pages/code/top-sloc.py
index db0c8592..4cdf2762 100644
--- a/api/pages/code/top-sloc.py
+++ b/api/pages/code/top-sloc.py
@@ -56,7 +56,7 @@
# security:
# - cookieAuth: []
# summary: Shows top 25 repos by lines of code
-#
+#
########################################################################
@@ -72,21 +72,21 @@
import re
def run(API, environ, indata, session):
-
+
# We need to be logged in for this!
if not session.user:
raise API.exception(403, "You must be logged in to use this API endpoint! %s")
-
+
now = time.time()
-
+
# First, fetch the view if we have such a thing enabled
viewList = []
if indata.get('view'):
viewList = session.getView(indata.get('view'))
if indata.get('subfilter'):
- viewList = session.subFilter(indata.get('subfilter'), view = viewList)
-
-
+ viewList = session.subFilter(indata.get('subfilter'), view = viewList)
+
+
####################################################################
####################################################################
dOrg = session.user['defaultOrganisation'] or "apache"
@@ -113,14 +113,14 @@ def run(API, environ, indata, session):
query['query']['bool']['must'].append({'term': {'sourceID': indata.get('source')}})
elif viewList:
query['query']['bool']['must'].append({'terms': {'sourceID': viewList}})
-
+
res = session.DB.ES.search(
index=session.DB.dbname,
doc_type="source",
size = 5000,
body = query
)
-
+
toprepos = []
for doc in res['hits']['hits']:
repo = doc['_source']
@@ -130,7 +130,7 @@ def run(API, environ, indata, session):
if not count:
count = 0
toprepos.append([url, count])
-
+
toprepos = sorted(toprepos, key = lambda x: int(x[1]), reverse = True)
top = toprepos[0:24]
if len(toprepos) > 25:
@@ -138,11 +138,11 @@ def run(API, environ, indata, session):
for repo in toprepos[25:]:
count += repo[1]
top.append(["Other repos", count])
-
+
tophash = {}
for v in top:
tophash[v[0]] = v[1]
-
+
JSON_OUT = {
'counts': tophash,
'okay': True,
diff --git a/api/pages/code/trends.py b/api/pages/code/trends.py
index 69d9b130..d0cfc449 100644
--- a/api/pages/code/trends.py
+++ b/api/pages/code/trends.py
@@ -56,7 +56,7 @@
# security:
# - cookieAuth: []
# summary: Shows trend data for a set of repos over a given period of time
-#
+#
########################################################################
@@ -71,29 +71,29 @@
import time
def run(API, environ, indata, session):
-
+
# We need to be logged in for this!
if not session.user:
raise API.exception(403, "You must be logged in to use this API endpoint! %s")
-
+
now = time.time()
-
+
# First, fetch the view if we have such a thing enabled
viewList = []
if indata.get('view'):
viewList = session.getView(indata.get('view'))
if indata.get('subfilter'):
- viewList = session.subFilter(indata.get('subfilter'), view = viewList)
-
-
+ viewList = session.subFilter(indata.get('subfilter'), view = viewList)
+
+
dateTo = indata.get('to', int(time.time()))
dateFrom = indata.get('from', dateTo - (86400*30*6)) # Default to a 6 month span
if dateFrom < 0:
dateFrom = 0
dateYonder = dateFrom - (dateTo - dateFrom)
-
-
-
+
+
+
####################################################################
# We start by doing all the queries for THIS period. #
# Then we reset the query, and change date to yonder-->from #
@@ -129,7 +129,7 @@ def run(API, environ, indata, session):
if indata.get('email'):
query['query']['bool']['should'] = [{'term': {'committer_email': indata.get('email')}}, {'term': {'author_email': indata.get('email')}}]
query['query']['bool']['minimum_should_match'] = 1
-
+
# Path filter?
if indata.get('pathfilter'):
pf = indata.get('pathfilter')
@@ -139,7 +139,7 @@ def run(API, environ, indata, session):
query['query']['bool']['must_not'].append({'regexp': {'files_changed': pf}})
else:
query['query']['bool']['must'].append({'regexp': {'files_changed': pf}})
-
+
# Get number of commits, this period
res = session.DB.ES.count(
index=session.DB.dbname,
@@ -147,8 +147,8 @@ def run(API, environ, indata, session):
body = query
)
no_commits = res['count']
-
-
+
+
# Get number of committers, this period
query['aggs'] = {
'commits': {
@@ -161,7 +161,7 @@ def run(API, environ, indata, session):
'field': 'author_email'
}
}
-
+
}
res = session.DB.ES.search(
index=session.DB.dbname,
@@ -171,8 +171,8 @@ def run(API, environ, indata, session):
)
no_committers = res['aggregations']['commits']['value']
no_authors = res['aggregations']['authors']['value']
-
-
+
+
# Get number of insertions, this period
query['aggs'] = {
'changes': {
@@ -188,7 +188,7 @@ def run(API, environ, indata, session):
body = query
)
insertions = res['aggregations']['changes']['value']
-
+
# Get number of deletions, this period
query['aggs'] = {
'changes': {
@@ -204,8 +204,8 @@ def run(API, environ, indata, session):
body = query
)
deletions = res['aggregations']['changes']['value']
-
-
+
+
####################################################################
# Change to PRIOR SPAN #
####################################################################
@@ -236,7 +236,7 @@ def run(API, environ, indata, session):
query['query']['bool']['must'].append({'term': {'sourceID': indata.get('source')}})
elif viewList:
query['query']['bool']['must'].append({'terms': {'sourceID': viewList}})
-
+
# Path filter?
if indata.get('pathfilter'):
pf = indata.get('pathfilter')
@@ -246,8 +246,8 @@ def run(API, environ, indata, session):
query['query']['bool']['must_not'].append({'regexp': {'files_changed': pf}})
else:
query['query']['bool']['must'].append({'regexp': {'files_changed': pf}})
-
-
+
+
# Get number of commits, this period
res = session.DB.ES.count(
index=session.DB.dbname,
@@ -255,7 +255,7 @@ def run(API, environ, indata, session):
body = query
)
no_commits_before = res['count']
-
+
# Get number of committers, this period
query['aggs'] = {
'commits': {
@@ -277,7 +277,7 @@ def run(API, environ, indata, session):
)
no_committers_before = res['aggregations']['commits']['value']
no_authors_before = res['aggregations']['authors']['value']
-
+
# Get number of insertions, this period
query['aggs'] = {
'changes': {
@@ -293,7 +293,7 @@ def run(API, environ, indata, session):
body = query
)
insertions_before = res['aggregations']['changes']['value']
-
+
# Get number of deletions, this period
query['aggs'] = {
'changes': {
@@ -309,9 +309,9 @@ def run(API, environ, indata, session):
body = query
)
deletions_before = res['aggregations']['changes']['value']
-
-
-
+
+
+
trends = {
"committers": {
'before': no_committers_before,
@@ -334,7 +334,7 @@ def run(API, environ, indata, session):
'title': "Lines changed this period"
}
}
-
+
JSON_OUT = {
'trends': trends,
'okay': True,
@@ -359,4 +359,3 @@ def run(API, environ, indata, session):
title = "Lines changed"
}
"""
-
\ No newline at end of file
diff --git a/api/pages/filters.py b/api/pages/filters.py
index d2960517..a97112cb 100644
--- a/api/pages/filters.py
+++ b/api/pages/filters.py
@@ -24,11 +24,11 @@
import time
def run(API, environ, indata, session):
-
+
# We need to be logged in for this!
if not session.user:
raise API.exception(403, "You must be logged in to use this API endpoint! %s")
-
+
# Fetch all sources for default org
dOrg = session.user['defaultOrganisation'] or "apache"
res = session.DB.ES.search(
@@ -56,7 +56,7 @@ def run(API, environ, indata, session):
sources.append(xdoc)
else:
sources.append(doc)
-
+
JSON_OUT = {
'views': sources,
'okay': True,
diff --git a/api/pages/forum/actors.py b/api/pages/forum/actors.py
index 345f59ae..40ad8ae2 100644
--- a/api/pages/forum/actors.py
+++ b/api/pages/forum/actors.py
@@ -56,7 +56,7 @@
# security:
# - cookieAuth: []
# summary: Shows timeseries of no. of people opening topics or replying to them.
-#
+#
########################################################################
@@ -72,27 +72,27 @@
import hashlib
def run(API, environ, indata, session):
-
+
# We need to be logged in for this!
if not session.user:
raise API.exception(403, "You must be logged in to use this API endpoint! %s")
-
+
now = time.time()
-
+
# First, fetch the view if we have such a thing enabled
viewList = []
if indata.get('view'):
viewList = session.getView(indata.get('view'))
if indata.get('subfilter'):
- viewList = session.subFilter(indata.get('subfilter'), view = viewList)
-
-
+ viewList = session.subFilter(indata.get('subfilter'), view = viewList)
+
+
dateTo = indata.get('to', int(time.time()))
dateFrom = indata.get('from', dateTo - (86400*30*6)) # Default to a 6 month span
-
+
interval = indata.get('interval', 'month')
-
-
+
+
####################################################################
####################################################################
dOrg = session.user['defaultOrganisation'] or "apache"
@@ -124,7 +124,7 @@ def run(API, environ, indata, session):
query['query']['bool']['must'].append({'terms': {'sourceID': viewList}})
if indata.get('email'):
query['query']['bool']['should'] = [{'term': {'issueCreator': indata.get('email')}}]
-
+
# Get timeseries for this period
query['aggs'] = {
'per_interval': {
@@ -141,14 +141,14 @@ def run(API, environ, indata, session):
}
}
}
-
+
res = session.DB.ES.search(
index=session.DB.dbname,
doc_type="forum_post",
size = 0,
body = query
)
-
+
timeseries = {}
for bucket in res['aggregations']['per_interval']['buckets']:
@@ -159,8 +159,8 @@ def run(API, environ, indata, session):
'topic responders': ccount,
'topic creators': 0
}
-
-
+
+
####################################################################
####################################################################
dOrg = session.user['defaultOrganisation'] or "apache"
@@ -192,7 +192,7 @@ def run(API, environ, indata, session):
query['query']['bool']['must'].append({'terms': {'sourceID': viewList}})
if indata.get('email'):
query['query']['bool']['should'] = [{'term': {'creator': indata.get('email')}}]
-
+
# Get timeseries for this period
query['aggs'] = {
'per_interval': {
@@ -209,7 +209,7 @@ def run(API, environ, indata, session):
}
}
}
-
+
res = session.DB.ES.search(
index=session.DB.dbname,
doc_type="forum_topic",
@@ -228,11 +228,11 @@ def run(API, environ, indata, session):
'topic creators': 0,
'topic responders': ccount
}
-
+
ts = []
for x, el in timeseries.items():
ts.append(el)
-
+
JSON_OUT = {
'timeseries': ts,
'okay': True,
diff --git a/api/pages/forum/creators.py b/api/pages/forum/creators.py
index dc6a6c6a..574e4577 100644
--- a/api/pages/forum/creators.py
+++ b/api/pages/forum/creators.py
@@ -56,7 +56,7 @@
# security:
# - cookieAuth: []
# summary: Shows the top N of forum topic creators
-#
+#
########################################################################
@@ -72,27 +72,27 @@
import hashlib
def run(API, environ, indata, session):
-
+
# We need to be logged in for this!
if not session.user:
raise API.exception(403, "You must be logged in to use this API endpoint! %s")
-
+
now = time.time()
-
+
# First, fetch the view if we have such a thing enabled
viewList = []
if indata.get('view'):
viewList = session.getView(indata.get('view'))
if indata.get('subfilter'):
- viewList = session.subFilter(indata.get('subfilter'), view = viewList)
-
-
+ viewList = session.subFilter(indata.get('subfilter'), view = viewList)
+
+
dateTo = indata.get('to', int(time.time()))
dateFrom = indata.get('from', dateTo - (86400*30*6)) # Default to a 6 month span
-
+
interval = indata.get('interval', 'month')
xtitle = None
-
+
####################################################################
####################################################################
dOrg = session.user['defaultOrganisation'] or "apache"
@@ -125,7 +125,7 @@ def run(API, environ, indata, session):
if indata.get('email'):
query['query']['bool']['must'].append({'term': {'creator': indata.get('email')}})
xtitle = "People opening issues solved by %s" % indata.get('email')
-
+
# Get top 25 committers this period
query['aggs'] = {
'committers': {
@@ -134,9 +134,9 @@ def run(API, environ, indata, session):
'size': 25
},
'aggs': {
-
+
}
- }
+ }
}
res = session.DB.ES.search(
index=session.DB.dbname,
@@ -161,7 +161,7 @@ def run(API, environ, indata, session):
people[email] = person
people[email]['gravatar'] = hashlib.md5(person.get('email', 'unknown').encode('utf-8')).hexdigest()
people[email]['count'] = count
-
+
topN = []
for email, person in people.items():
topN.append(person)
diff --git a/api/pages/forum/issues.py b/api/pages/forum/issues.py
index a485bfac..fa27035b 100644
--- a/api/pages/forum/issues.py
+++ b/api/pages/forum/issues.py
@@ -56,7 +56,7 @@
# security:
# - cookieAuth: []
# summary: Shows timeseries of forum topics opened/responded-to over time
-#
+#
########################################################################
@@ -81,40 +81,40 @@ def makeTS(dist):
return ts
def run(API, environ, indata, session):
-
+
# We need to be logged in for this!
if not session.user:
raise API.exception(403, "You must be logged in to use this API endpoint! %s")
-
+
now = time.time()
-
+
# First, fetch the view if we have such a thing enabled
viewList = []
if indata.get('view'):
viewList = session.getView(indata.get('view'))
if indata.get('subfilter'):
- viewList = session.subFilter(indata.get('subfilter'), view = viewList)
-
-
+ viewList = session.subFilter(indata.get('subfilter'), view = viewList)
+
+
dateTo = indata.get('to', int(time.time()))
dateFrom = indata.get('from', dateTo - (86400*30*6)) # Default to a 6 month span
-
+
interval = indata.get('interval', 'month')
-
+
# By default, we lump generic forums and question/answer (like SO, askbot) together as one
distinct = {
'forum': ['discourse', 'stackoverflow', 'askbot']
}
-
+
# If requested, we split them into two
if indata.get('distinguish', False):
distinct = {
'forum': ['discourse'],
'question bank': ['stackoverflow', 'askbot']
}
-
+
timeseries = {}
-
+
# For each category and the issue types that go along with that,
# grab opened and closed over time.
for iType, iValues in distinct.items():
@@ -155,14 +155,14 @@ def run(API, environ, indata, session):
query['query']['bool']['must'].append({'terms': {'sourceID': viewList}})
if indata.get('email'):
query['query']['bool']['must'].append({'term': {'creator': indata.get('email')}})
-
+
# Get number of opened ones, this period
query['aggs'] = {
'commits': {
'date_histogram': {
'field': 'createdDate',
'interval': interval
- }
+ }
}
}
res = session.DB.ES.search(
@@ -171,14 +171,14 @@ def run(API, environ, indata, session):
size = 0,
body = query
)
-
+
for bucket in res['aggregations']['commits']['buckets']:
ts = int(bucket['key'] / 1000)
count = bucket['doc_count']
timeseries[ts] = timeseries.get(ts, makeTS(distinct))
timeseries[ts][iType + ' topics'] = timeseries[ts].get(iType + ' topics', 0) + count
-
-
+
+
####################################################################
# ISSUES CLOSED #
####################################################################
@@ -215,14 +215,14 @@ def run(API, environ, indata, session):
query['query']['bool']['must'].append({'term': {'sourceID': indata.get('source')}})
if indata.get('email'):
query['query']['bool']['must'].append({'term': {'creator': indata.get('email')}})
-
+
# Get number of closed ones, this period
query['aggs'] = {
'commits': {
'date_histogram': {
'field': 'createdDate',
'interval': interval
- }
+ }
}
}
res = session.DB.ES.search(
@@ -231,19 +231,19 @@ def run(API, environ, indata, session):
size = 0,
body = query
)
-
+
for bucket in res['aggregations']['commits']['buckets']:
ts = int(bucket['key'] / 1000)
count = bucket['doc_count']
timeseries[ts] = timeseries.get(ts, makeTS(distinct))
timeseries[ts][iType + ' replies'] = timeseries[ts].get(iType + ' replies', 0) + count
-
+
ts = []
for k, v in timeseries.items():
v['date'] = k
ts.append(v)
-
-
+
+
JSON_OUT = {
'widgetType': {
'chartType': 'line', # Recommendation for the UI
diff --git a/api/pages/forum/responders.py b/api/pages/forum/responders.py
index 6c12ca2a..d379c1ef 100644
--- a/api/pages/forum/responders.py
+++ b/api/pages/forum/responders.py
@@ -56,7 +56,7 @@
# security:
# - cookieAuth: []
# summary: Shows the top N of issue closers
-#
+#
########################################################################
@@ -72,28 +72,28 @@
import hashlib
def run(API, environ, indata, session):
-
+
# We need to be logged in for this!
if not session.user:
raise API.exception(403, "You must be logged in to use this API endpoint! %s")
-
+
now = time.time()
-
+
# First, fetch the view if we have such a thing enabled
viewList = []
if indata.get('view'):
viewList = session.getView(indata.get('view'))
if indata.get('subfilter'):
- viewList = session.subFilter(indata.get('subfilter'), view = viewList)
-
-
+ viewList = session.subFilter(indata.get('subfilter'), view = viewList)
+
+
dateTo = indata.get('to', int(time.time()))
dateFrom = indata.get('from', dateTo - (86400*30*6)) # Default to a 6 month span
-
+
interval = indata.get('interval', 'month')
xtitle = None
-
-
+
+
####################################################################
####################################################################
dOrg = session.user['defaultOrganisation'] or "apache"
@@ -126,7 +126,7 @@ def run(API, environ, indata, session):
if indata.get('email'):
query['query']['bool']['must'].append({'term': {'creator': indata.get('email')}})
xTitle = "People closing %s's issues" % indata.get('email')
-
+
# Get top 25 committers this period
query['aggs'] = {
'committers': {
@@ -135,9 +135,9 @@ def run(API, environ, indata, session):
'size': 25
},
'aggs': {
-
+
}
- }
+ }
}
res = session.DB.ES.search(
index=session.DB.dbname,
@@ -162,7 +162,7 @@ def run(API, environ, indata, session):
people[email] = person
people[email]['gravatar'] = hashlib.md5(person.get('email', 'unknown').encode('utf-8')).hexdigest()
people[email]['count'] = count
-
+
topN = []
for email, person in people.items():
topN.append(person)
diff --git a/api/pages/forum/top-count.py b/api/pages/forum/top-count.py
index 58d345c1..585407fd 100644
--- a/api/pages/forum/top-count.py
+++ b/api/pages/forum/top-count.py
@@ -56,7 +56,7 @@
# security:
# - cookieAuth: []
# summary: Shows top 25 forums by interactions
-#
+#
########################################################################
@@ -72,24 +72,24 @@
import re
def run(API, environ, indata, session):
-
+
# We need to be logged in for this!
if not session.user:
raise API.exception(403, "You must be logged in to use this API endpoint! %s")
-
+
now = time.time()
-
+
# First, fetch the view if we have such a thing enabled
viewList = []
if indata.get('view'):
viewList = session.getView(indata.get('view'))
if indata.get('subfilter'):
- viewList = session.subFilter(indata.get('subfilter'), view = viewList)
-
-
+ viewList = session.subFilter(indata.get('subfilter'), view = viewList)
+
+
dateTo = indata.get('to', int(time.time()))
dateFrom = indata.get('from', dateTo - (86400*30*6)) # Default to a 6 month span
-
+
####################################################################
####################################################################
dOrg = session.user['defaultOrganisation'] or "apache"
@@ -123,15 +123,15 @@ def run(API, environ, indata, session):
query['query']['bool']['should'] = [
{'term': {'creator': indata.get('email')}}
]
-
-
+
+
# Get top 25 committers this period
query['aggs'] = {
'by_repo': {
'terms': {
'field': 'sourceID',
'size': 5000
- }
+ }
}
}
res = session.DB.ES.search(
@@ -140,7 +140,7 @@ def run(API, environ, indata, session):
size = 0,
body = query
)
-
+
toprepos = []
for bucket in res['aggregations']['by_repo']['buckets']:
ID = bucket['key']
@@ -149,7 +149,7 @@ def run(API, environ, indata, session):
repo = re.sub(r".+/([^/]+)$", r"\1", it['sourceURL'])
count = bucket['doc_count']
toprepos.append([repo, count])
-
+
toprepos = sorted(toprepos, key = lambda x: x[1], reverse = True)
top = toprepos[0:24]
if len(toprepos) > 25:
@@ -157,11 +157,11 @@ def run(API, environ, indata, session):
for repo in toprepos[25:]:
count += repo[1]
top.append(["Other forums", count])
-
+
tophash = {}
for v in top:
tophash[v[0]] = v[1]
-
+
JSON_OUT = {
'counts': tophash,
'okay': True,
diff --git a/api/pages/forum/top.py b/api/pages/forum/top.py
index 51d4c8d6..3d184320 100644
--- a/api/pages/forum/top.py
+++ b/api/pages/forum/top.py
@@ -56,7 +56,7 @@
# security:
# - cookieAuth: []
# summary: Shows the top N topics by interactions
-#
+#
########################################################################
@@ -72,27 +72,27 @@
import hashlib
def run(API, environ, indata, session):
-
+
# We need to be logged in for this!
if not session.user:
raise API.exception(403, "You must be logged in to use this API endpoint! %s")
-
+
now = time.time()
-
+
# First, fetch the view if we have such a thing enabled
viewList = []
if indata.get('view'):
viewList = session.getView(indata.get('view'))
if indata.get('subfilter'):
- viewList = session.subFilter(indata.get('subfilter'), view = viewList)
-
-
+ viewList = session.subFilter(indata.get('subfilter'), view = viewList)
+
+
dateTo = indata.get('to', int(time.time()))
dateFrom = indata.get('from', dateTo - (86400*30*6)) # Default to a 6 month span
-
+
interval = indata.get('interval', 'month')
-
-
+
+
####################################################################
####################################################################
dOrg = session.user['defaultOrganisation'] or "apache"
@@ -127,7 +127,7 @@ def run(API, environ, indata, session):
query['query']['bool']['must'].append({'terms': {'sourceID': viewList}})
if indata.get('email'):
query['query']['bool']['should'] = [{'term': {'creator': indata.get('email')}}]
-
+
res = session.DB.ES.search(
index=session.DB.dbname,
doc_type="forum_topic",
@@ -142,8 +142,8 @@ def run(API, environ, indata, session):
doc['subject'] = doc.get('title')
doc['count'] = doc.get('posts', 0)
top.append(doc)
-
-
+
+
JSON_OUT = {
'topN': {
'denoter': 'interactions',
diff --git a/api/pages/forum/trends.py b/api/pages/forum/trends.py
index 6012890e..208ccb0f 100644
--- a/api/pages/forum/trends.py
+++ b/api/pages/forum/trends.py
@@ -56,7 +56,7 @@
# security:
# - cookieAuth: []
# summary: Shows trend data for a set of forums over a given period of time
-#
+#
########################################################################
@@ -71,30 +71,30 @@
import time
def run(API, environ, indata, session):
-
+
# We need to be logged in for this!
if not session.user:
raise API.exception(403, "You must be logged in to use this API endpoint! %s")
-
+
now = time.time()
-
+
# First, fetch the view if we have such a thing enabled
viewList = []
if indata.get('view'):
viewList = session.getView(indata.get('view'))
if indata.get('subfilter'):
- viewList = session.subFilter(indata.get('subfilter'), view = viewList)
-
-
+ viewList = session.subFilter(indata.get('subfilter'), view = viewList)
+
+
dateTo = indata.get('to', int(time.time()))
dateFrom = indata.get('from', dateTo - (86400*30*6)) # Default to a 6 month span
if dateFrom < 0:
dateFrom = 0
dateYonder = dateFrom - (dateTo - dateFrom)
-
-
+
+
dOrg = session.user['defaultOrganisation'] or "apache"
-
+
####################################################################
# We start by doing all the queries for THIS period. #
# Then we reset the query, and change date to yonder-->from #
@@ -126,7 +126,7 @@ def run(API, environ, indata, session):
query['query']['bool']['must'].append({'term': {'sourceID': indata.get('source')}})
elif viewList:
query['query']['bool']['must'].append({'terms': {'sourceID': viewList}})
-
+
# Get number of issues created, this period
res = session.DB.ES.count(
index=session.DB.dbname,
@@ -134,8 +134,8 @@ def run(API, environ, indata, session):
body = query
)
no_issues_created = res['count']
-
-
+
+
# Get number of open/close, this period
query['aggs'] = {
'opener': {
@@ -151,10 +151,10 @@ def run(API, environ, indata, session):
body = query
)
no_creators = res['aggregations']['opener']['value']
-
-
+
+
# REPLIERS
-
+
query = {
'query': {
'bool': {
@@ -182,7 +182,7 @@ def run(API, environ, indata, session):
elif viewList:
query['query']['bool']['must'].append({'terms': {'sourceID': viewList}})
-
+
# Get number of issues created, this period
res = session.DB.ES.count(
index=session.DB.dbname,
@@ -190,8 +190,8 @@ def run(API, environ, indata, session):
body = query
)
no_issues_closed = res['count']
-
-
+
+
# Get number of open/close, this period
query['aggs'] = {
'closer': {
@@ -207,8 +207,8 @@ def run(API, environ, indata, session):
body = query
)
no_closers = res['aggregations']['closer']['value']
-
-
+
+
####################################################################
# Change to PRIOR SPAN #
####################################################################
@@ -233,13 +233,13 @@ def run(API, environ, indata, session):
}
}
}
-
+
if indata.get('source'):
query['query']['bool']['must'].append({'term': {'sourceID': indata.get('source')}})
elif viewList:
query['query']['bool']['must'].append({'terms': {'sourceID': viewList}})
-
-
+
+
# Get number of issues, this period
res = session.DB.ES.count(
index=session.DB.dbname,
@@ -247,7 +247,7 @@ def run(API, environ, indata, session):
body = query
)
no_issues_created_before = res['count']
-
+
# Get number of committers, this period
query['aggs'] = {
'opener': {
@@ -263,11 +263,11 @@ def run(API, environ, indata, session):
body = query
)
no_creators_before = res['aggregations']['opener']['value']
-
-
-
+
+
+
# REPLIERS
-
+
query = {
'query': {
'bool': {
@@ -293,7 +293,7 @@ def run(API, environ, indata, session):
query['query']['bool']['must'].append({'term': {'sourceID': indata.get('source')}})
elif viewList:
query['query']['bool']['must'].append({'terms': {'sourceID': viewList}})
-
+
# Get number of issues created, this period
res = session.DB.ES.count(
index=session.DB.dbname,
@@ -301,8 +301,8 @@ def run(API, environ, indata, session):
body = query
)
no_issues_closed_before = res['count']
-
-
+
+
# Get number of open/close, this period
query['aggs'] = {
'closer': {
@@ -318,7 +318,7 @@ def run(API, environ, indata, session):
body = query
)
no_closers_before = res['aggregations']['closer']['value']
-
+
trends = {
"created": {
'before': no_issues_created_before,
@@ -341,7 +341,7 @@ def run(API, environ, indata, session):
'title': "People replying this period"
}
}
-
+
JSON_OUT = {
'trends': trends,
'okay': True,
diff --git a/api/pages/issue/actors.py b/api/pages/issue/actors.py
index 37a5124a..b53e839a 100644
--- a/api/pages/issue/actors.py
+++ b/api/pages/issue/actors.py
@@ -56,7 +56,7 @@
# security:
# - cookieAuth: []
# summary: Shows timeseries of no. of people opening/closing issues over time
-#
+#
########################################################################
@@ -72,27 +72,27 @@
import hashlib
def run(API, environ, indata, session):
-
+
# We need to be logged in for this!
if not session.user:
raise API.exception(403, "You must be logged in to use this API endpoint! %s")
-
+
now = time.time()
-
+
# First, fetch the view if we have such a thing enabled
viewList = []
if indata.get('view'):
viewList = session.getView(indata.get('view'))
if indata.get('subfilter'):
- viewList = session.subFilter(indata.get('subfilter'), view = viewList)
-
-
+ viewList = session.subFilter(indata.get('subfilter'), view = viewList)
+
+
dateTo = indata.get('to', int(time.time()))
dateFrom = indata.get('from', dateTo - (86400*30*6)) # Default to a 6 month span
-
+
interval = indata.get('interval', 'month')
-
-
+
+
####################################################################
####################################################################
dOrg = session.user['defaultOrganisation'] or "apache"
@@ -125,7 +125,7 @@ def run(API, environ, indata, session):
if indata.get('email'):
query['query']['bool']['should'] = [{'term': {'issueCreator': indata.get('email')}}, {'term': {'issueCloser': indata.get('email')}}]
query['query']['bool']['minimum_should_match'] = 1
-
+
# Get timeseries for this period
query['aggs'] = {
'per_interval': {
@@ -142,14 +142,14 @@ def run(API, environ, indata, session):
}
}
}
-
+
res = session.DB.ES.search(
index=session.DB.dbname,
doc_type="issue",
size = 0,
body = query
)
-
+
timeseries = {}
for bucket in res['aggregations']['per_interval']['buckets']:
ts = int(bucket['key'] / 1000)
@@ -159,8 +159,8 @@ def run(API, environ, indata, session):
'closers': ccount,
'openers': 0
}
-
-
+
+
####################################################################
####################################################################
dOrg = session.user['defaultOrganisation'] or "apache"
@@ -193,7 +193,7 @@ def run(API, environ, indata, session):
if indata.get('email'):
query['query']['bool']['should'] = [{'term': {'issueCreator': indata.get('email')}}, {'term': {'issueCloser': indata.get('email')}}]
query['query']['bool']['minimum_should_match'] = 1
-
+
# Get timeseries for this period
query['aggs'] = {
'per_interval': {
@@ -210,7 +210,7 @@ def run(API, environ, indata, session):
}
}
}
-
+
res = session.DB.ES.search(
index=session.DB.dbname,
doc_type="issue",
@@ -229,11 +229,11 @@ def run(API, environ, indata, session):
'closers': 0,
'openers': ccount
}
-
+
ts = []
for x, el in timeseries.items():
ts.append(el)
-
+
JSON_OUT = {
'timeseries': ts,
'okay': True,
diff --git a/api/pages/issue/age.py b/api/pages/issue/age.py
index 4ab1f617..b6a3d901 100644
--- a/api/pages/issue/age.py
+++ b/api/pages/issue/age.py
@@ -56,7 +56,7 @@
# security:
# - cookieAuth: []
# summary: Shows timeseries of no. of open tickets by age
-#
+#
########################################################################
@@ -72,23 +72,23 @@
import hashlib
def run(API, environ, indata, session):
-
+
# We need to be logged in for this!
if not session.user:
raise API.exception(403, "You must be logged in to use this API endpoint! %s")
-
+
now = time.time()
-
+
# First, fetch the view if we have such a thing enabled
viewList = []
if indata.get('view'):
viewList = session.getView(indata.get('view'))
if indata.get('subfilter'):
- viewList = session.subFilter(indata.get('subfilter'), view = viewList)
-
-
+ viewList = session.subFilter(indata.get('subfilter'), view = viewList)
+
+
interval = indata.get('interval', 'month')
-
+
####################################################################
####################################################################
dOrg = session.user['defaultOrganisation'] or "apache"
@@ -118,7 +118,7 @@ def run(API, environ, indata, session):
if indata.get('email'):
query['query']['bool']['should'] = [{'term': {'issueCreator': indata.get('email')}}, {'term': {'issueCloser': indata.get('email')}}]
query['query']['bool']['minimum_should_match'] = 1
-
+
# Get timeseries for this period
query['aggs'] = {
'per_interval': {
@@ -128,7 +128,7 @@ def run(API, environ, indata, session):
}
}
}
-
+
res = session.DB.ES.search(
index=session.DB.dbname,
doc_type="issue",
@@ -144,9 +144,9 @@ def run(API, environ, indata, session):
'date': ts,
'open': opened
})
-
-
-
+
+
+
JSON_OUT = {
'timeseries': timeseries,
'okay': True,
diff --git a/api/pages/issue/closers.py b/api/pages/issue/closers.py
index 6515130e..53ada24a 100644
--- a/api/pages/issue/closers.py
+++ b/api/pages/issue/closers.py
@@ -56,7 +56,7 @@
# security:
# - cookieAuth: []
# summary: Shows the top N of issue closers
-#
+#
########################################################################
@@ -72,28 +72,28 @@
import hashlib
def run(API, environ, indata, session):
-
+
# We need to be logged in for this!
if not session.user:
raise API.exception(403, "You must be logged in to use this API endpoint! %s")
-
+
now = time.time()
-
+
# First, fetch the view if we have such a thing enabled
viewList = []
if indata.get('view'):
viewList = session.getView(indata.get('view'))
if indata.get('subfilter'):
- viewList = session.subFilter(indata.get('subfilter'), view = viewList)
-
-
+ viewList = session.subFilter(indata.get('subfilter'), view = viewList)
+
+
dateTo = indata.get('to', int(time.time()))
dateFrom = indata.get('from', dateTo - (86400*30*6)) # Default to a 6 month span
-
+
interval = indata.get('interval', 'month')
xtitle = None
-
-
+
+
####################################################################
####################################################################
dOrg = session.user['defaultOrganisation'] or "apache"
@@ -126,7 +126,7 @@ def run(API, environ, indata, session):
if indata.get('email'):
query['query']['bool']['must'].append({'term': {'issueCreator': indata.get('email')}})
xTitle = "People closing %s's issues" % indata.get('email')
-
+
# Get top 25 committers this period
query['aggs'] = {
'committers': {
@@ -135,9 +135,9 @@ def run(API, environ, indata, session):
'size': 25
},
'aggs': {
-
+
}
- }
+ }
}
res = session.DB.ES.search(
index=session.DB.dbname,
@@ -162,7 +162,7 @@ def run(API, environ, indata, session):
people[email] = person
people[email]['gravatar'] = hashlib.md5(person.get('email', 'unknown').encode('utf-8')).hexdigest()
people[email]['count'] = count
-
+
topN = []
for email, person in people.items():
topN.append(person)
diff --git a/api/pages/issue/issues.py b/api/pages/issue/issues.py
index 623eaa7e..dac17211 100644
--- a/api/pages/issue/issues.py
+++ b/api/pages/issue/issues.py
@@ -56,7 +56,7 @@
# security:
# - cookieAuth: []
# summary: Shows timeseries of issues opened/closed over time
-#
+#
########################################################################
@@ -81,40 +81,40 @@ def makeTS(dist):
return ts
def run(API, environ, indata, session):
-
+
# We need to be logged in for this!
if not session.user:
raise API.exception(403, "You must be logged in to use this API endpoint! %s")
-
+
now = time.time()
-
+
# First, fetch the view if we have such a thing enabled
viewList = []
if indata.get('view'):
viewList = session.getView(indata.get('view'))
if indata.get('subfilter'):
- viewList = session.subFilter(indata.get('subfilter'), view = viewList)
-
-
+ viewList = session.subFilter(indata.get('subfilter'), view = viewList)
+
+
dateTo = indata.get('to', int(time.time()))
dateFrom = indata.get('from', dateTo - (86400*30*6)) # Default to a 6 month span
-
+
interval = indata.get('interval', 'month')
-
+
# By default, we lump PRs and issues into the same category
distinct = {
'issues': ['issue', 'pullrequest']
}
-
+
# If requested, we split them into two
if indata.get('distinguish', False):
distinct = {
'issues': ['issue'],
'pull requests': ['pullrequest']
}
-
+
timeseries = {}
-
+
# For each category and the issue types that go along with that,
# grab opened and closed over time.
for iType, iValues in distinct.items():
@@ -155,14 +155,14 @@ def run(API, environ, indata, session):
query['query']['bool']['must'].append({'terms': {'sourceID': viewList}})
if indata.get('email'):
query['query']['bool']['must'].append({'term': {'issueCreator': indata.get('email')}})
-
+
# Get number of opened ones, this period
query['aggs'] = {
'commits': {
'date_histogram': {
'field': 'createdDate',
'interval': interval
- }
+ }
}
}
res = session.DB.ES.search(
@@ -171,14 +171,14 @@ def run(API, environ, indata, session):
size = 0,
body = query
)
-
+
for bucket in res['aggregations']['commits']['buckets']:
ts = int(bucket['key'] / 1000)
count = bucket['doc_count']
timeseries[ts] = timeseries.get(ts, makeTS(distinct))
timeseries[ts][iType + ' opened'] = timeseries[ts].get(iType + ' opened', 0) + count
-
-
+
+
####################################################################
# ISSUES CLOSED #
####################################################################
@@ -215,14 +215,14 @@ def run(API, environ, indata, session):
query['query']['bool']['must'].append({'term': {'sourceID': indata.get('source')}})
if indata.get('email'):
query['query']['bool']['must'].append({'term': {'issueCloser': indata.get('email')}})
-
+
# Get number of closed ones, this period
query['aggs'] = {
'commits': {
'date_histogram': {
'field': 'closedDate',
'interval': interval
- }
+ }
}
}
res = session.DB.ES.search(
@@ -231,19 +231,19 @@ def run(API, environ, indata, session):
size = 0,
body = query
)
-
+
for bucket in res['aggregations']['commits']['buckets']:
ts = int(bucket['key'] / 1000)
count = bucket['doc_count']
timeseries[ts] = timeseries.get(ts, makeTS(distinct))
timeseries[ts][iType + ' closed'] = timeseries[ts].get(iType + ' closed', 0) + count
-
+
ts = []
for k, v in timeseries.items():
v['date'] = k
ts.append(v)
-
-
+
+
JSON_OUT = {
'widgetType': {
'chartType': 'line', # Recommendation for the UI
diff --git a/api/pages/issue/openers.py b/api/pages/issue/openers.py
index 321e5d0f..a2e081e5 100644
--- a/api/pages/issue/openers.py
+++ b/api/pages/issue/openers.py
@@ -56,7 +56,7 @@
# security:
# - cookieAuth: []
# summary: Shows the top N of issue openers
-#
+#
########################################################################
@@ -72,27 +72,27 @@
import hashlib
def run(API, environ, indata, session):
-
+
# We need to be logged in for this!
if not session.user:
raise API.exception(403, "You must be logged in to use this API endpoint! %s")
-
+
now = time.time()
-
+
# First, fetch the view if we have such a thing enabled
viewList = []
if indata.get('view'):
viewList = session.getView(indata.get('view'))
if indata.get('subfilter'):
- viewList = session.subFilter(indata.get('subfilter'), view = viewList)
-
-
+ viewList = session.subFilter(indata.get('subfilter'), view = viewList)
+
+
dateTo = indata.get('to', int(time.time()))
dateFrom = indata.get('from', dateTo - (86400*30*6)) # Default to a 6 month span
-
+
interval = indata.get('interval', 'month')
xtitle = None
-
+
####################################################################
####################################################################
dOrg = session.user['defaultOrganisation'] or "apache"
@@ -125,7 +125,7 @@ def run(API, environ, indata, session):
if indata.get('email'):
query['query']['bool']['must'].append({'term': {'issueCloser': indata.get('email')}})
xtitle = "People opening issues solved by %s" % indata.get('email')
-
+
# Get top 25 committers this period
query['aggs'] = {
'committers': {
@@ -134,9 +134,9 @@ def run(API, environ, indata, session):
'size': 25
},
'aggs': {
-
+
}
- }
+ }
}
res = session.DB.ES.search(
index=session.DB.dbname,
@@ -161,7 +161,7 @@ def run(API, environ, indata, session):
people[email] = person
people[email]['gravatar'] = hashlib.md5(person.get('email', 'unknown').encode('utf-8')).hexdigest()
people[email]['count'] = count
-
+
topN = []
for email, person in people.items():
topN.append(person)
diff --git a/api/pages/issue/pony-timeseries.py b/api/pages/issue/pony-timeseries.py
index 2bf096d5..f8ae3608 100644
--- a/api/pages/issue/pony-timeseries.py
+++ b/api/pages/issue/pony-timeseries.py
@@ -56,7 +56,7 @@
# security:
# - cookieAuth: []
# summary: Shows timeseries of Pony Factor over time
-#
+#
########################################################################
@@ -74,31 +74,31 @@
import dateutil.relativedelta
def run(API, environ, indata, session):
-
+
# We need to be logged in for this!
if not session.user:
raise API.exception(403, "You must be logged in to use this API endpoint! %s")
-
+
now = time.time()
-
+
# First, fetch the view if we have such a thing enabled
viewList = []
if indata.get('view'):
viewList = session.getView(indata.get('view'))
if indata.get('subfilter'):
- viewList = session.subFilter(indata.get('subfilter'), view = viewList)
-
-
+ viewList = session.subFilter(indata.get('subfilter'), view = viewList)
+
+
hl = indata.get('span', 24)
tnow = datetime.date.today()
nm = tnow.month - (tnow.month % 3)
ny = tnow.year
ts = []
-
+
if nm < 1:
nm += 12
ny = ny - 1
-
+
while ny > 1970:
d = datetime.date(ny, nm, 1)
t = time.mktime(d.timetuple())
@@ -108,8 +108,8 @@ def run(API, environ, indata, session):
if nm < 1:
nm += 12
ny = ny - 1
-
-
+
+
####################################################################
####################################################################
dOrg = session.user['defaultOrganisation'] or "apache"
@@ -139,31 +139,31 @@ def run(API, environ, indata, session):
query['query']['bool']['must'].append({'term': {'sourceID': indata.get('source')}})
elif viewList:
query['query']['bool']['must'].append({'terms': {'sourceID': viewList}})
-
+
# Get an initial count of commits
res = session.DB.ES.count(
index=session.DB.dbname,
doc_type="issue",
body = query
)
-
+
globcount = res['count']
if globcount == 0:
break
-
+
# Get top 25 committers this period
query['aggs'] = {
'by_creator': {
'terms': {
'field': 'issueCreator',
'size': 1000
- }
+ }
},
'by_closer': {
'terms': {
'field': 'issueCloser',
'size': 1000
- }
+ }
}
}
res = session.DB.ES.search(
@@ -172,9 +172,9 @@ def run(API, environ, indata, session):
size = 0,
body = query
)
-
+
cpf = {}
-
+
# PF for openers
pf_opener = 0
pf_opener_count = 0
@@ -187,7 +187,7 @@ def run(API, environ, indata, session):
cpf[mldom] = True
if pf_opener_count > int(globcount/2):
break
-
+
# PF for closer
pf_closer = 0
pf_closer_count = 0
@@ -206,9 +206,9 @@ def run(API, environ, indata, session):
'Pony Factor (closers)': pf_closer,
'Meta-Pony Factor': len(cpf)
})
-
+
ts = sorted(ts, key = lambda x: x['date'])
-
+
JSON_OUT = {
'text': "This shows Pony Factors as calculated over a %u month timespan. Openers measures the people submitting the bulk of the issues, closers mesaures the people closing (resolving) the issues, and meta-pony is an estimation of how many organisations/companies are involved." % hl,
'timeseries': ts,
diff --git a/api/pages/issue/relationships.py b/api/pages/issue/relationships.py
index f660832f..ff0c9d6a 100644
--- a/api/pages/issue/relationships.py
+++ b/api/pages/issue/relationships.py
@@ -56,7 +56,7 @@
# security:
# - cookieAuth: []
# summary: Shows a breakdown of contributor relationships between issue trackers
-#
+#
########################################################################
@@ -75,32 +75,32 @@
import math
def run(API, environ, indata, session):
-
+
# We need to be logged in for this!
if not session.user:
raise API.exception(403, "You must be logged in to use this API endpoint! %s")
-
+
now = time.time()
-
+
# First, fetch the view if we have such a thing enabled
viewList = []
if indata.get('view'):
viewList = session.getView(indata.get('view'))
if indata.get('subfilter'):
- viewList = session.subFilter(indata.get('subfilter'), view = viewList)
-
+ viewList = session.subFilter(indata.get('subfilter'), view = viewList)
+
dateTo = indata.get('to', int(time.time()))
dateFrom = indata.get('from', dateTo - (86400*30*6)) # Default to a 6 month span
-
+
which = 'committer_email'
role = 'committer'
if indata.get('author', False):
which = 'author_email'
role = 'author'
-
+
interval = indata.get('interval', 'day')
-
-
+
+
####################################################################
####################################################################
dOrg = session.user['defaultOrganisation'] or "apache"
@@ -133,14 +133,14 @@ def run(API, environ, indata, session):
if indata.get('email'):
query['query']['bool']['should'] = [{'term': {'issueCreator': indata.get('email')}}, {'term': {'issueCloser': indata.get('email')}}]
query['query']['bool']['minimum_should_match'] = 1
-
+
# Get number of commits, this period, per repo
query['aggs'] = {
'per_repo': {
'terms': {
'field': 'sourceID',
'size': 10000
- }
+ }
}
}
res = session.DB.ES.search(
@@ -149,7 +149,7 @@ def run(API, environ, indata, session):
size = 0,
body = query
)
-
+
repos = {}
repo_commits = {}
authorlinks = {}
@@ -157,25 +157,25 @@ def run(API, environ, indata, session):
max_links = 0
max_shared = 0
max_authors = 0
-
+
# For each repo, count commits and gather data on authors
for doc in res['aggregations']['per_repo']['buckets']:
sourceID = doc['key']
commits = doc['doc_count']
-
+
# Gather the unique authors/committers
query['aggs'] = {
'per_closer': {
'terms': {
'field': 'issueCloser',
'size': 10000
- }
+ }
},
'per_creator': {
'terms': {
'field': 'issueCreator',
'size': 10000
- }
+ }
}
}
xquery = copy.deepcopy(query)
@@ -195,7 +195,7 @@ def run(API, environ, indata, session):
max_commits = commits
repos[sourceID] = authors
repo_commits[sourceID] = commits
-
+
# Now, figure out which repos share the same contributors
repo_links = {}
repo_notoriety = {}
@@ -209,7 +209,7 @@ def run(API, environ, indata, session):
if not session.DB.ES.exists(index=session.DB.dbname, doc_type="source", id = ID):
continue
repodatas[ID] = session.DB.ES.get(index=session.DB.dbname, doc_type="source", id = ID)
-
+
for ID, repo in repos.items():
mylinks = {}
if not ID in repodatas:
@@ -246,11 +246,11 @@ def run(API, environ, indata, session):
if ID not in repo_notoriety:
repo_notoriety[ID] = set()
repo_notoriety[ID].update(mylinks.keys()) # How many projects is this repo connected to?
-
+
if ID not in repo_authors:
repo_authors[ID] = set()
repo_authors[ID].update(repo) # How many projects is this repo connected to?
-
+
if ID != oID:
repo_commits[ID] = repo_commits.get(ID, 0) + repo_commits[oID]
if repo_commits[ID] > max_commits:
@@ -259,7 +259,7 @@ def run(API, environ, indata, session):
max_links = len(repo_notoriety[ID])
if len(repo_authors[ID]) > max_authors:
max_authors = len(repo_authors[ID]) # Used for calculating max sphere size in charts
-
+
# Now, pull it all together!
nodes = []
links = []
@@ -282,7 +282,7 @@ def run(API, environ, indata, session):
}
nodes.append(doc)
existing_repos.append(sourceID)
-
+
for k, s in repo_links.items():
size = s
fr, to = k.split('@')
@@ -295,7 +295,7 @@ def run(API, environ, indata, session):
'tooltip': "%u contributors in common" % size
}
links.append(doc)
-
+
JSON_OUT = {
'maxLinks': max_links,
'maxShared': max_shared,
diff --git a/api/pages/issue/retention.py b/api/pages/issue/retention.py
index 22e021e3..3d745418 100644
--- a/api/pages/issue/retention.py
+++ b/api/pages/issue/retention.py
@@ -58,7 +58,7 @@
# - cookieAuth: []
# summary: Shows retention metrics for a set of issue trackers over a given period
# of time
-#
+#
########################################################################
@@ -75,37 +75,37 @@
import datetime
def run(API, environ, indata, session):
-
+
# We need to be logged in for this!
if not session.user:
raise API.exception(403, "You must be logged in to use this API endpoint! %s")
-
+
now = time.time()
-
+
# First, fetch the view if we have such a thing enabled
viewList = []
if indata.get('view'):
viewList = session.getView(indata.get('view'))
if indata.get('subfilter'):
- viewList = session.subFilter(indata.get('subfilter'), view = viewList)
-
-
+ viewList = session.subFilter(indata.get('subfilter'), view = viewList)
+
+
hl = indata.get('span', 12) # By default, we define a contributor as active if having committer in the past year
tnow = datetime.date.today()
nm = tnow.month - (tnow.month % 3)
ny = tnow.year
cy = ny
ts = []
-
+
if nm < 1:
nm += 12
ny = ny - 1
-
+
peopleSeen = {}
activePeople = {}
allPeople = {}
FoundSomething = False
-
+
ny = 1970
while ny < cy or (ny == cy and (nm+3) <= tnow.month):
d = datetime.date(ny, nm, 1)
@@ -118,7 +118,7 @@ def run(API, environ, indata, session):
break
d = datetime.date(ny, nm, 1)
tf = time.mktime(d.timetuple())
-
+
####################################################################
####################################################################
dOrg = session.user['defaultOrganisation'] or "apache"
@@ -148,32 +148,32 @@ def run(API, environ, indata, session):
query['query']['bool']['must'].append({'term': {'sourceID': indata.get('source')}})
elif viewList:
query['query']['bool']['must'].append({'terms': {'sourceID': viewList}})
-
+
# Get an initial count of commits
res = session.DB.ES.count(
index=session.DB.dbname,
doc_type="issue",
body = query
)
-
+
globcount = res['count']
if globcount == 0 and FoundSomething == False:
continue
FoundSomething = True
-
+
# Get top 1000 committers this period
query['aggs'] = {
'by_o': {
'terms': {
'field': 'issueCloser',
'size': 50000
- }
+ }
},
'by_c': {
'terms': {
'field': 'issueCreator',
'size': 50000
- }
+ }
}
}
res = session.DB.ES.search(
@@ -182,12 +182,12 @@ def run(API, environ, indata, session):
size = 0,
body = query
)
-
-
+
+
retained = 0
added = 0
lost = 0
-
+
thisPeriod = []
for bucket in res['aggregations']['by_o']['buckets']:
who = bucket['key']
@@ -198,7 +198,7 @@ def run(API, environ, indata, session):
activePeople[who] = tf
if who not in allPeople:
allPeople[who] = tf
-
+
for bucket in res['aggregations']['by_c']['buckets']:
who = bucket['key']
thisPeriod.append(who)
@@ -209,13 +209,13 @@ def run(API, environ, indata, session):
activePeople[who] = tf
if who not in allPeople:
allPeople[who] = tf
-
+
prune = []
for k, v in activePeople.items():
if v < (t - (hl*30.45*86400)):
prune.append(k)
lost += 1
-
+
for who in prune:
del activePeople[who]
del peopleSeen[who]
@@ -227,14 +227,14 @@ def run(API, environ, indata, session):
'People retained': retained,
'Active people': added + retained
})
-
+
groups = [
['More than 5 years', (5*365*86400)+1],
['2 - 5 years', (2*365*86400)+1],
['1 - 2 years', (365*86400)],
['Less than a year', 1]
]
-
+
counts = {}
totExp = 0
for person, age in activePeople.items():
@@ -244,9 +244,9 @@ def run(API, environ, indata, session):
counts[el[0]] = counts.get(el[0], 0) + 1
break
avgyr = (totExp / (86400*365)) / max(len(activePeople),1)
-
+
ts = sorted(ts, key = lambda x: x['date'])
-
+
avgm = ""
yr = int(avgyr)
ym = round((avgyr-yr)*12)
diff --git a/api/pages/issue/top-count.py b/api/pages/issue/top-count.py
index f17f721b..ebb121c8 100644
--- a/api/pages/issue/top-count.py
+++ b/api/pages/issue/top-count.py
@@ -56,7 +56,7 @@
# security:
# - cookieAuth: []
# summary: Shows top 25 issue trackers by issues
-#
+#
########################################################################
@@ -72,24 +72,24 @@
import re
def run(API, environ, indata, session):
-
+
# We need to be logged in for this!
if not session.user:
raise API.exception(403, "You must be logged in to use this API endpoint! %s")
-
+
now = time.time()
-
+
# First, fetch the view if we have such a thing enabled
viewList = []
if indata.get('view'):
viewList = session.getView(indata.get('view'))
if indata.get('subfilter'):
- viewList = session.subFilter(indata.get('subfilter'), view = viewList)
-
-
+ viewList = session.subFilter(indata.get('subfilter'), view = viewList)
+
+
dateTo = indata.get('to', int(time.time()))
dateFrom = indata.get('from', dateTo - (86400*30*6)) # Default to a 6 month span
-
+
####################################################################
####################################################################
dOrg = session.user['defaultOrganisation'] or "apache"
@@ -125,15 +125,15 @@ def run(API, environ, indata, session):
{'term': {'issueCloser': indata.get('email')}}
]
query['query']['bool']['minimum_should_match'] = 1
-
-
+
+
# Get top 25 committers this period
query['aggs'] = {
'by_repo': {
'terms': {
'field': 'sourceID',
'size': 5000
- }
+ }
}
}
res = session.DB.ES.search(
@@ -142,7 +142,7 @@ def run(API, environ, indata, session):
size = 0,
body = query
)
-
+
toprepos = []
for bucket in res['aggregations']['by_repo']['buckets']:
ID = bucket['key']
@@ -151,7 +151,7 @@ def run(API, environ, indata, session):
repo = re.sub(r".+/([^/]+)$", r"\1", it['sourceURL'])
count = bucket['doc_count']
toprepos.append([repo, count])
-
+
toprepos = sorted(toprepos, key = lambda x: x[1], reverse = True)
top = toprepos[0:24]
if len(toprepos) > 25:
@@ -159,11 +159,11 @@ def run(API, environ, indata, session):
for repo in toprepos[25:]:
count += repo[1]
top.append(["Other trackers", count])
-
+
tophash = {}
for v in top:
tophash[v[0]] = v[1]
-
+
JSON_OUT = {
'counts': tophash,
'okay': True,
diff --git a/api/pages/issue/top.py b/api/pages/issue/top.py
index 33cde721..42b4d38e 100644
--- a/api/pages/issue/top.py
+++ b/api/pages/issue/top.py
@@ -56,7 +56,7 @@
# security:
# - cookieAuth: []
# summary: Shows the top N issues by interactions
-#
+#
########################################################################
@@ -72,27 +72,27 @@
import hashlib
def run(API, environ, indata, session):
-
+
# We need to be logged in for this!
if not session.user:
raise API.exception(403, "You must be logged in to use this API endpoint! %s")
-
+
now = time.time()
-
+
# First, fetch the view if we have such a thing enabled
viewList = []
if indata.get('view'):
viewList = session.getView(indata.get('view'))
if indata.get('subfilter'):
- viewList = session.subFilter(indata.get('subfilter'), view = viewList)
-
-
+ viewList = session.subFilter(indata.get('subfilter'), view = viewList)
+
+
dateTo = indata.get('to', int(time.time()))
dateFrom = indata.get('from', dateTo - (86400*30*6)) # Default to a 6 month span
-
+
interval = indata.get('interval', 'month')
-
-
+
+
####################################################################
####################################################################
dOrg = session.user['defaultOrganisation'] or "apache"
@@ -128,7 +128,7 @@ def run(API, environ, indata, session):
if indata.get('email'):
query['query']['bool']['should'] = [{'term': {'issueCreator': indata.get('email')}}, {'term': {'issueCloser': indata.get('email')}}]
query['query']['bool']['minimum_should_match'] = 1
-
+
res = session.DB.ES.search(
index=session.DB.dbname,
doc_type="issue",
@@ -143,8 +143,8 @@ def run(API, environ, indata, session):
doc['subject'] = doc.get('title')
doc['count'] = doc.get('comments', 0)
top.append(doc)
-
-
+
+
JSON_OUT = {
'topN': {
'denoter': 'interactions',
diff --git a/api/pages/issue/trends.py b/api/pages/issue/trends.py
index 7387d88f..c6b45152 100644
--- a/api/pages/issue/trends.py
+++ b/api/pages/issue/trends.py
@@ -56,7 +56,7 @@
# security:
# - cookieAuth: []
# summary: Shows trend data for a set of issue trackers over a given period of time
-#
+#
########################################################################
@@ -71,30 +71,30 @@
import time
def run(API, environ, indata, session):
-
+
# We need to be logged in for this!
if not session.user:
raise API.exception(403, "You must be logged in to use this API endpoint! %s")
-
+
now = time.time()
-
+
# First, fetch the view if we have such a thing enabled
viewList = []
if indata.get('view'):
viewList = session.getView(indata.get('view'))
if indata.get('subfilter'):
- viewList = session.subFilter(indata.get('subfilter'), view = viewList)
-
-
+ viewList = session.subFilter(indata.get('subfilter'), view = viewList)
+
+
dateTo = indata.get('to', int(time.time()))
dateFrom = indata.get('from', dateTo - (86400*30*6)) # Default to a 6 month span
if dateFrom < 0:
dateFrom = 0
dateYonder = dateFrom - (dateTo - dateFrom)
-
-
+
+
dOrg = session.user['defaultOrganisation'] or "apache"
-
+
####################################################################
# We start by doing all the queries for THIS period. #
# Then we reset the query, and change date to yonder-->from #
@@ -129,7 +129,7 @@ def run(API, environ, indata, session):
if indata.get('email'):
query['query']['bool']['should'] = [{'term': {'issueCreator': indata.get('email')}}, {'term': {'issueCloser': indata.get('email')}}]
query['query']['bool']['minimum_should_match'] = 1
-
+
# Get number of issues created, this period
res = session.DB.ES.count(
index=session.DB.dbname,
@@ -137,8 +137,8 @@ def run(API, environ, indata, session):
body = query
)
no_issues_created = res['count']
-
-
+
+
# Get number of open/close, this period
query['aggs'] = {
'opener': {
@@ -154,10 +154,10 @@ def run(API, environ, indata, session):
body = query
)
no_creators = res['aggregations']['opener']['value']
-
-
+
+
# CLOSERS
-
+
query = {
'query': {
'bool': {
@@ -187,7 +187,7 @@ def run(API, environ, indata, session):
if indata.get('email'):
query['query']['bool']['should'] = [{'term': {'issueCreator': indata.get('email')}}, {'term': {'issueCloser': indata.get('email')}}]
query['query']['bool']['minimum_should_match'] = 1
-
+
# Get number of issues created, this period
res = session.DB.ES.count(
index=session.DB.dbname,
@@ -195,8 +195,8 @@ def run(API, environ, indata, session):
body = query
)
no_issues_closed = res['count']
-
-
+
+
# Get number of open/close, this period
query['aggs'] = {
'closer': {
@@ -212,9 +212,9 @@ def run(API, environ, indata, session):
body = query
)
no_closers = res['aggregations']['closer']['value']
-
-
-
+
+
+
####################################################################
# Change to PRIOR SPAN #
####################################################################
@@ -244,7 +244,7 @@ def run(API, environ, indata, session):
if indata.get('email'):
query['query']['bool']['should'] = [{'term': {'issueCreator': indata.get('email')}}, {'term': {'issueCloser': indata.get('email')}}]
query['query']['bool']['minimum_should_match'] = 1
-
+
# Get number of issues, this period
res = session.DB.ES.count(
index=session.DB.dbname,
@@ -252,7 +252,7 @@ def run(API, environ, indata, session):
body = query
)
no_issues_created_before = res['count']
-
+
# Get number of committers, this period
query['aggs'] = {
'opener': {
@@ -268,11 +268,11 @@ def run(API, environ, indata, session):
body = query
)
no_creators_before = res['aggregations']['opener']['value']
-
-
-
+
+
+
# CLOSERS
-
+
query = {
'query': {
'bool': {
@@ -299,7 +299,7 @@ def run(API, environ, indata, session):
if indata.get('email'):
query['query']['bool']['should'] = [{'term': {'issueCreator': indata.get('email')}}, {'term': {'issueCloser': indata.get('email')}}]
query['query']['bool']['minimum_should_match'] = 1
-
+
# Get number of issues created, this period
res = session.DB.ES.count(
index=session.DB.dbname,
@@ -307,8 +307,8 @@ def run(API, environ, indata, session):
body = query
)
no_issues_closed_before = res['count']
-
-
+
+
# Get number of open/close, this period
query['aggs'] = {
'closer': {
@@ -324,8 +324,8 @@ def run(API, environ, indata, session):
body = query
)
no_closers_before = res['aggregations']['closer']['value']
-
-
+
+
trends = {
"created": {
'before': no_issues_created_before,
@@ -348,7 +348,7 @@ def run(API, environ, indata, session):
'title': "People closing issues this period"
}
}
-
+
JSON_OUT = {
'trends': trends,
'okay': True,
diff --git a/api/pages/mail/map.py b/api/pages/mail/map.py
index 3c446cc8..b0f8398c 100644
--- a/api/pages/mail/map.py
+++ b/api/pages/mail/map.py
@@ -56,7 +56,7 @@
# security:
# - cookieAuth: []
# summary: Shows a breakdown of email author reply mappings
-#
+#
########################################################################
@@ -77,24 +77,24 @@
badBots = r"(JIRA|Hudson|jira|jenkins|GitHub|git@|dev@|bugzilla|gerrit)"
def run(API, environ, indata, session):
-
+
# We need to be logged in for this!
if not session.user:
raise API.exception(403, "You must be logged in to use this API endpoint! %s")
-
+
now = time.time()
-
+
# First, fetch the view if we have such a thing enabled
viewList = []
if indata.get('view'):
viewList = session.getView(indata.get('view'))
if indata.get('subfilter'):
- viewList = session.subFilter(indata.get('subfilter'), view = viewList)
-
+ viewList = session.subFilter(indata.get('subfilter'), view = viewList)
+
dateTo = indata.get('to', int(time.time()))
dateFrom = indata.get('from', dateTo - (86400*30*6)) # Default to a 6 month span
span = dateTo - dateFrom
-
+
####################################################################
####################################################################
dOrg = session.user['defaultOrganisation'] or "apache"
@@ -126,21 +126,21 @@ def run(API, environ, indata, session):
query['query']['bool']['must'].append({'terms': {'sourceID': viewList}})
if indata.get('search'):
query['query']['bool']['must'].append({'regexp': {'subject': indata.get('search')}})
-
+
if indata.get('email'):
query['query']['bool']['minimum_should_match'] = 1
query['query']['bool']['should'] = [
{'term': {'replyto.keyword': indata.get('email')}},
{'term': {'sender': indata.get('email')}},
]
-
+
# Get number of commits, this period, per repo
query['aggs'] = {
'per_ml': {
'terms': {
'field': 'replyto.keyword' if not indata.get('author') else 'sender',
'size': 150
- }
+ }
}
}
res = session.DB.ES.search(
@@ -149,7 +149,7 @@ def run(API, environ, indata, session):
size = 0,
body = query
)
-
+
repos = {}
repo_commits = {}
authorlinks = {}
@@ -158,11 +158,11 @@ def run(API, environ, indata, session):
max_shared = 0
max_authors = 0
minLinks = indata.get('links', 1)
-
+
if indata.get('email'):
del query['query']['bool']['should']
del query['query']['bool']['minimum_should_match']
-
+
# For each repo, count commits and gather data on authors
for doc in res['aggregations']['per_ml']['buckets']:
sourceID = doc['key']
@@ -171,19 +171,19 @@ def run(API, environ, indata, session):
continue
if emails > (span/86400)*4: # More than 4/day and we consider you a bot!
continue
-
-
+
+
# Gather the unique authors/committers
query['aggs'] = {
'per_ml': {
'terms': {
'field': 'sender' if not indata.get('author') else 'replyto.keyword',
'size': 5000
- }
+ }
}
}
xquery = copy.deepcopy(query)
-
+
xquery['query']['bool']['must'].append({'term': {'replyto.keyword' if not indata.get('author') else 'sender': sourceID}})
xres = session.DB.ES.search(
index=session.DB.dbname,
@@ -199,7 +199,7 @@ def run(API, environ, indata, session):
max_emails = emails
repos[sourceID] = authors
repo_commits[sourceID] = emails
-
+
# Now, figure out which repos share the same contributors
repo_links = {}
repo_notoriety = {}
@@ -213,7 +213,7 @@ def run(API, environ, indata, session):
if not session.DB.ES.exists(index=session.DB.dbname, doc_type="person", id = hID):
continue
repodatas[ID] = session.DB.ES.get(index=session.DB.dbname, doc_type="person", id = hID)
-
+
for ID, repo in repos.items():
mylinks = {}
if not ID in repodatas:
@@ -233,7 +233,7 @@ def run(API, environ, indata, session):
if m:
xID = m.group(1)
if xID != ID:
-
+
if ID in xrepo:
xlinks.append(xID)
lname = "%s||%s" % (ID, xID) # Link name
@@ -248,11 +248,11 @@ def run(API, environ, indata, session):
if ID not in repo_notoriety:
repo_notoriety[ID] = set()
repo_notoriety[ID].update(mylinks.keys()) # How many projects is this repo connected to?
-
+
if ID not in repo_authors:
repo_authors[ID] = set()
repo_authors[ID].update(repo) # How many projects is this repo connected to?
-
+
if ID != oID:
repo_commits[ID] = repo_commits.get(ID, 0) + repo_commits[oID]
if repo_commits[ID] > max_emails:
@@ -261,7 +261,7 @@ def run(API, environ, indata, session):
max_links = len(repo_notoriety[ID])
if len(repo_authors[ID]) > max_authors:
max_authors = len(repo_authors[ID]) # Used for calculating max sphere size in charts
-
+
# Now, pull it all together!
nodes = []
links = []
@@ -285,7 +285,7 @@ def run(API, environ, indata, session):
}
nodes.append(doc)
existing_repos.append(sourceID)
-
+
for k, s in repo_links.items():
size = s
fr, to = k.split('||')
@@ -298,7 +298,7 @@ def run(API, environ, indata, session):
'tooltip': "%u topics exchanged" % size
}
links.append(doc)
-
+
JSON_OUT = {
'maxLinks': max_links,
'maxShared': max_shared,
diff --git a/api/pages/mail/mood-timeseries.py b/api/pages/mail/mood-timeseries.py
index be7e3b28..21f26d03 100644
--- a/api/pages/mail/mood-timeseries.py
+++ b/api/pages/mail/mood-timeseries.py
@@ -56,7 +56,7 @@
# security:
# - cookieAuth: []
# summary: Shows a breakdown of the (analyzed) mood in emails as a timeseries
-#
+#
########################################################################
@@ -71,29 +71,29 @@
import time
def run(API, environ, indata, session):
-
+
# We need to be logged in for this!
if not session.user:
raise API.exception(403, "You must be logged in to use this API endpoint! %s")
-
-
+
+
# First, fetch the view if we have such a thing enabled
viewList = []
if indata.get('view'):
viewList = session.getView(indata.get('view'))
if indata.get('subfilter'):
- viewList = session.subFilter(indata.get('subfilter'), view = viewList)
-
+ viewList = session.subFilter(indata.get('subfilter'), view = viewList)
+
dateTo = indata.get('to', int(time.time()))
dateFrom = indata.get('from', dateTo - (86400*30*6)) # Default to a 6 month span
interval = indata.get('interval', 'week')
-
+
# Define moods we know of
moods_good = set(['trust', 'joy', 'confident', 'positive'])
moods_bad = set(['sadness', 'anger', 'disgust', 'fear', 'negative'])
moods_neutral = set(['anticipation', 'surprise', 'tentative', 'analytical', 'neutral'])
all_moods = set(moods_good | moods_bad | moods_neutral)
-
+
# Fetch all sources for default org
dOrg = session.user['defaultOrganisation'] or "apache"
query = {
@@ -126,13 +126,13 @@ def run(API, environ, indata, session):
query['query']['bool']['must'].append({'term': {'sourceID': indata.get('source')}})
elif viewList:
query['query']['bool']['must'].append({'terms': {'sourceID': viewList}})
-
+
emls = session.DB.ES.count(
index=session.DB.dbname,
doc_type="email",
body = query
)['count']
-
+
query['aggs'] = {
'history': {
'date_histogram': {
@@ -143,26 +143,26 @@ def run(API, environ, indata, session):
}
}
}
-
+
# Add aggregations for moods
for mood in all_moods:
query['aggs']['history']['aggs'][mood] = {
'sum': {
'field': "mood.%s" % mood
- }
+ }
}
-
-
+
+
res = session.DB.ES.search(
index=session.DB.dbname,
doc_type="email",
size = 0,
body = query
)
-
+
timeseries = []
-
-
+
+
for tz in res['aggregations']['history']['buckets']:
moods = {}
emls = tz['doc_count']
@@ -170,7 +170,7 @@ def run(API, environ, indata, session):
moods[mood] = int (100 * tz.get(mood, {'value':0})['value'] / max(1, emls))
moods['date'] = int(tz['key']/1000)
timeseries.append(moods)
-
+
JSON_OUT = {
'timeseries': timeseries,
'okay': True
diff --git a/api/pages/mail/mood.py b/api/pages/mail/mood.py
index a1beb46b..38dd57fc 100644
--- a/api/pages/mail/mood.py
+++ b/api/pages/mail/mood.py
@@ -56,7 +56,7 @@
# security:
# - cookieAuth: []
# summary: Shows a breakdown of the (analyzed) mood in emails
-#
+#
########################################################################
@@ -71,28 +71,28 @@
import time
def run(API, environ, indata, session):
-
+
# We need to be logged in for this!
if not session.user:
raise API.exception(403, "You must be logged in to use this API endpoint! %s")
-
-
+
+
# First, fetch the view if we have such a thing enabled
viewList = []
if indata.get('view'):
viewList = session.getView(indata.get('view'))
if indata.get('subfilter'):
- viewList = session.subFilter(indata.get('subfilter'), view = viewList)
-
+ viewList = session.subFilter(indata.get('subfilter'), view = viewList)
+
dateTo = indata.get('to', int(time.time()))
dateFrom = indata.get('from', dateTo - (86400*30*6)) # Default to a 6 month span
-
+
# Define moods we know of
moods_good = set(['trust', 'joy', 'confident', 'positive'])
moods_bad = set(['sadness', 'anger', 'disgust', 'fear', 'negative'])
moods_neutral = set(['anticipation', 'surprise', 'tentative', 'analytical', 'neutral'])
all_moods = set(moods_good | moods_bad | moods_neutral)
-
+
# Start off with a query for the entire org (we want to compare)
dOrg = session.user['defaultOrganisation'] or "apache"
query = {
@@ -120,26 +120,26 @@ def run(API, environ, indata, session):
}
}
}
-
+
# Count all emails, for averaging scores
gemls = session.DB.ES.count(
index=session.DB.dbname,
doc_type="email",
body = query
)['count']
-
+
# Add aggregations for moods
query['aggs'] = {
-
+
}
for mood in all_moods:
query['aggs'][mood] = {
'sum': {
'field': "mood.%s" % mood
- }
+ }
}
-
-
+
+
global_mood_compiled = {}
mood_compiled = {}
txt = "This chart shows the ten potential mood types as they average on the emails in this period. A score of 100 means a sentiment is highly visible in most emails."
@@ -150,7 +150,7 @@ def run(API, environ, indata, session):
txt = "This chart shows the ten potential mood types on the selected lists as they compare against all mailing lists in the database. A score of 100 here means the sentiment conforms to averages across all lists."
gtxt = "This shows the overall estimated mood compared to all lists, as a gauge from terrible to good."
global_moods = {}
-
+
gres = session.DB.ES.search(
index=session.DB.dbname,
doc_type="email",
@@ -165,7 +165,7 @@ def run(API, environ, indata, session):
for k, v in global_moods.items():
if v >= 0:
global_mood_compiled[k] = int( (v / max(1,gemls)) * 100)
-
+
# Now, if we have a view (or not distinguishing), ...
ss = False
if indata.get('source'):
@@ -174,7 +174,7 @@ def run(API, environ, indata, session):
elif viewList:
query['query']['bool']['must'].append({'terms': {'sourceID': viewList}})
ss = True
-
+
# If we have a view enabled (and distinguish), compile local view against global view
# Else, just copy global as local
if ss or not indata.get('relative'):
@@ -184,17 +184,17 @@ def run(API, environ, indata, session):
size = 0,
body = query
)
-
+
del query['aggs'] # we have to remove these to do a count()
emls = session.DB.ES.count(
index=session.DB.dbname,
doc_type="email",
body = query
)['count']
-
+
moods = {}
years = 0
-
+
for mood, el in res['aggregations'].items():
if el['value'] == 0:
el['value'] == -1
@@ -204,13 +204,13 @@ def run(API, environ, indata, session):
mood_compiled[k] = int(100 * int( ( v / max(1,emls)) * 100) / max(1, global_mood_compiled.get(k, 100)))
else:
mood_compiled = global_mood_compiled
-
+
# If relative mode and a field is missing, assume 100 (norm)
if indata.get('relative'):
for M in all_moods:
if mood_compiled.get(M, 0) == 0:
mood_compiled[M] = 100
-
+
# Compile an overall happiness level
MAX = max(max(mood_compiled.values()),1)
X = 100 if indata.get('relative') else 0
@@ -218,9 +218,9 @@ def run(API, environ, indata, session):
for B in moods_bad:
if mood_compiled.get(B) and mood_compiled[B] > X:
bads += mood_compiled[B]
-
+
happ = 50
-
+
goods = X
for B in moods_good:
if mood_compiled.get(B) and mood_compiled[B] > X:
@@ -231,7 +231,7 @@ def run(API, environ, indata, session):
if goods > 0:
happ += (50*goods/MAX)
swingometer = max(0, min(100, happ))
-
+
# JSON out!
JSON_OUT = {
'relativeMode': True,
diff --git a/api/pages/mail/pony-timeseries.py b/api/pages/mail/pony-timeseries.py
index fefd7762..37d160fb 100644
--- a/api/pages/mail/pony-timeseries.py
+++ b/api/pages/mail/pony-timeseries.py
@@ -56,7 +56,7 @@
# security:
# - cookieAuth: []
# summary: Shows timeseries of Pony Factor over time
-#
+#
########################################################################
@@ -74,30 +74,30 @@
import dateutil.relativedelta
def run(API, environ, indata, session):
-
+
# We need to be logged in for this!
if not session.user:
raise API.exception(403, "You must be logged in to use this API endpoint! %s")
-
+
now = time.time()
-
+
# First, fetch the view if we have such a thing enabled
viewList = []
if indata.get('view'):
viewList = session.getView(indata.get('view'))
if indata.get('subfilter'):
- viewList = session.subFilter(indata.get('subfilter'), view = viewList)
-
+ viewList = session.subFilter(indata.get('subfilter'), view = viewList)
+
hl = indata.get('span', 24)
tnow = datetime.date.today()
nm = tnow.month - (tnow.month % 3)
ny = tnow.year
ts = []
-
+
if nm < 1:
nm += 12
ny = ny - 1
-
+
while ny > 1970:
d = datetime.date(ny, nm, 1)
t = time.mktime(d.timetuple())
@@ -107,8 +107,8 @@ def run(API, environ, indata, session):
if nm < 1:
nm += 12
ny = ny - 1
-
-
+
+
####################################################################
####################################################################
dOrg = session.user['defaultOrganisation'] or "apache"
@@ -145,25 +145,25 @@ def run(API, environ, indata, session):
query['query']['bool']['must'].append({'term': {'sourceID': indata.get('source')}})
elif viewList:
query['query']['bool']['must'].append({'terms': {'sourceID': viewList}})
-
+
# Get an initial count of commits
res = session.DB.ES.count(
index=session.DB.dbname,
doc_type="email",
body = query
)
-
+
globcount = res['count']
if globcount == 0:
break
-
+
# Get top 25 committers this period
query['aggs'] = {
'by_sender': {
'terms': {
'field': 'sender',
'size': 2500
- }
+ }
}
}
res = session.DB.ES.search(
@@ -172,8 +172,8 @@ def run(API, environ, indata, session):
size = 0,
body = query
)
-
-
+
+
# PF for authors
pf_author = 0
pf_author_count = 0
@@ -196,9 +196,9 @@ def run(API, environ, indata, session):
'Pony Factor (authors)': pf_author,
'Meta-Pony Factor': len(cpf)
})
-
+
ts = sorted(ts, key = lambda x: x['date'])
-
+
JSON_OUT = {
'text': "This shows Pony Factors as calculated over a %u month timespan. Authorship is a measure of the people it takes to make up the bulk of email traffic, and meta-pony is an estimation of how many organisations/companies are involved." % hl,
'timeseries': ts,
diff --git a/api/pages/mail/relationships.py b/api/pages/mail/relationships.py
index 16a0cdb0..c10b6343 100644
--- a/api/pages/mail/relationships.py
+++ b/api/pages/mail/relationships.py
@@ -56,7 +56,7 @@
# security:
# - cookieAuth: []
# summary: Shows a breakdown of contributor relationships between mailing lists
-#
+#
########################################################################
@@ -75,24 +75,24 @@
import math
def run(API, environ, indata, session):
-
+
# We need to be logged in for this!
if not session.user:
raise API.exception(403, "You must be logged in to use this API endpoint! %s")
-
+
now = time.time()
-
+
# First, fetch the view if we have such a thing enabled
viewList = []
if indata.get('view'):
viewList = session.getView(indata.get('view'))
if indata.get('subfilter'):
- viewList = session.subFilter(indata.get('subfilter'), view = viewList)
-
+ viewList = session.subFilter(indata.get('subfilter'), view = viewList)
+
dateTo = indata.get('to', int(time.time()))
dateFrom = indata.get('from', dateTo - (86400*30*6)) # Default to a 6 month span
-
-
+
+
####################################################################
####################################################################
dOrg = session.user['defaultOrganisation'] or "apache"
@@ -124,14 +124,14 @@ def run(API, environ, indata, session):
query['query']['bool']['must'].append({'terms': {'sourceID': viewList}})
if indata.get('email'):
query['query']['bool']['must'].append({'term': {'sender': indata.get('email')}})
-
+
# Get number of commits, this period, per repo
query['aggs'] = {
'per_ml': {
'terms': {
'field': 'sourceID',
'size': 10000
- }
+ }
}
}
res = session.DB.ES.search(
@@ -140,7 +140,7 @@ def run(API, environ, indata, session):
size = 0,
body = query
)
-
+
repos = {}
repo_commits = {}
authorlinks = {}
@@ -149,19 +149,19 @@ def run(API, environ, indata, session):
max_shared = 0
max_authors = 0
minLinks = indata.get('links', 1)
-
+
# For each repo, count commits and gather data on authors
for doc in res['aggregations']['per_ml']['buckets']:
sourceID = doc['key']
emails = doc['doc_count']
-
+
# Gather the unique authors/committers
query['aggs'] = {
'per_ml': {
'terms': {
'field': 'sender',
'size': 10000
- }
+ }
}
}
xquery = copy.deepcopy(query)
@@ -179,7 +179,7 @@ def run(API, environ, indata, session):
max_emails = emails
repos[sourceID] = authors
repo_commits[sourceID] = emails
-
+
# Now, figure out which repos share the same contributors
repo_links = {}
repo_notoriety = {}
@@ -192,7 +192,7 @@ def run(API, environ, indata, session):
if not session.DB.ES.exists(index=session.DB.dbname, doc_type="source", id = ID):
continue
repodatas[ID] = session.DB.ES.get(index=session.DB.dbname, doc_type="source", id = ID)
-
+
for ID, repo in repos.items():
mylinks = {}
if not ID in repodatas:
@@ -229,11 +229,11 @@ def run(API, environ, indata, session):
if ID not in repo_notoriety:
repo_notoriety[ID] = set()
repo_notoriety[ID].update(mylinks.keys()) # How many projects is this repo connected to?
-
+
if ID not in repo_authors:
repo_authors[ID] = set()
repo_authors[ID].update(repo) # How many projects is this repo connected to?
-
+
if ID != oID:
repo_commits[ID] = repo_commits.get(ID, 0) + repo_commits[oID]
if repo_commits[ID] > max_emails:
@@ -242,7 +242,7 @@ def run(API, environ, indata, session):
max_links = len(repo_notoriety[ID])
if len(repo_authors[ID]) > max_authors:
max_authors = len(repo_authors[ID]) # Used for calculating max sphere size in charts
-
+
# Now, pull it all together!
nodes = []
links = []
@@ -265,7 +265,7 @@ def run(API, environ, indata, session):
}
nodes.append(doc)
existing_repos.append(sourceID)
-
+
for k, s in repo_links.items():
size = s
fr, to = k.split('||')
@@ -278,7 +278,7 @@ def run(API, environ, indata, session):
'tooltip': "%u contributors in common" % size
}
links.append(doc)
-
+
JSON_OUT = {
'maxLinks': max_links,
'maxShared': max_shared,
diff --git a/api/pages/mail/retention.py b/api/pages/mail/retention.py
index 6734da11..bc93b9ce 100644
--- a/api/pages/mail/retention.py
+++ b/api/pages/mail/retention.py
@@ -58,7 +58,7 @@
# - cookieAuth: []
# summary: Shows retention metrics for a set of mailing lists over a given period
# of time
-#
+#
########################################################################
@@ -75,36 +75,36 @@
import datetime
def run(API, environ, indata, session):
-
+
# We need to be logged in for this!
if not session.user:
raise API.exception(403, "You must be logged in to use this API endpoint! %s")
-
+
now = time.time()
-
+
# First, fetch the view if we have such a thing enabled
viewList = []
if indata.get('view'):
viewList = session.getView(indata.get('view'))
if indata.get('subfilter'):
- viewList = session.subFilter(indata.get('subfilter'), view = viewList)
-
-
+ viewList = session.subFilter(indata.get('subfilter'), view = viewList)
+
+
hl = indata.get('span', 12) # By default, we define a contributor as active if having committer in the past year
tnow = datetime.date.today()
nm = tnow.month - (tnow.month % 3)
ny = tnow.year
cy = ny
ts = []
-
+
if nm < 1:
nm += 12
ny = ny - 1
-
+
peopleSeen = {}
activePeople = {}
allPeople = {}
-
+
ny = 1970
FoundSomething = False
while ny < cy or (ny == cy and (nm+3) <= tnow.month):
@@ -118,8 +118,8 @@ def run(API, environ, indata, session):
break
d = datetime.date(ny, nm, 1)
tf = time.mktime(d.timetuple())
-
-
+
+
####################################################################
####################################################################
dOrg = session.user['defaultOrganisation'] or "apache"
@@ -149,14 +149,14 @@ def run(API, environ, indata, session):
query['query']['bool']['must'].append({'term': {'sourceID': indata.get('source')}})
elif viewList:
query['query']['bool']['must'].append({'terms': {'sourceID': viewList}})
-
+
# Get an initial count of commits
res = session.DB.ES.count(
index=session.DB.dbname,
doc_type="email",
body = query
)
-
+
globcount = res['count']
if globcount == 0 and not FoundSomething:
continue
@@ -167,7 +167,7 @@ def run(API, environ, indata, session):
'terms': {
'field': 'sender',
'size': 200000
- }
+ }
}
}
res = session.DB.ES.search(
@@ -176,12 +176,12 @@ def run(API, environ, indata, session):
size = 0,
body = query
)
-
-
+
+
retained = 0
added = 0
lost = 0
-
+
thisPeriod = []
for bucket in res['aggregations']['by_author']['buckets']:
who = bucket['key']
@@ -192,18 +192,18 @@ def run(API, environ, indata, session):
activePeople[who] = tf
if who not in allPeople:
allPeople[who] = tf
-
+
prune = []
for k, v in activePeople.items():
if v < (t - (hl*30.45*86400)):
prune.append(k)
lost += 1
-
+
for who in prune:
del activePeople[who]
del peopleSeen[who]
retained = len(activePeople) - added
-
+
ts.append({
'date': tf,
'People who (re)joined': added,
@@ -211,14 +211,14 @@ def run(API, environ, indata, session):
'People retained': retained,
'Active people': added + retained
})
-
+
groups = [
['More than 5 years', (5*365*86400)+1],
['2 - 5 years', (2*365*86400)+1],
['1 - 2 years', (365*86400)],
['Less than a year', 1]
]
-
+
counts = {}
totExp = 0
for person, age in activePeople.items():
@@ -228,9 +228,9 @@ def run(API, environ, indata, session):
counts[el[0]] = counts.get(el[0], 0) + 1
break
avgyr = (totExp / (86400*365)) / max(len(activePeople),1)
-
+
ts = sorted(ts, key = lambda x: x['date'])
-
+
avgm = ""
yr = int(avgyr)
ym = round((avgyr-yr)*12)
diff --git a/api/pages/mail/timeseries-single.py b/api/pages/mail/timeseries-single.py
index 855539cc..8b8b231c 100644
--- a/api/pages/mail/timeseries-single.py
+++ b/api/pages/mail/timeseries-single.py
@@ -56,7 +56,7 @@
# security:
# - cookieAuth: []
# summary: Shows email sent over time
-#
+#
########################################################################
@@ -73,27 +73,27 @@
import hashlib
def run(API, environ, indata, session):
-
+
# We need to be logged in for this!
if not session.user:
raise API.exception(403, "You must be logged in to use this API endpoint! %s")
-
+
now = time.time()
-
+
# First, fetch the view if we have such a thing enabled
viewList = []
if indata.get('view'):
viewList = session.getView(indata.get('view'))
if indata.get('subfilter'):
- viewList = session.subFilter(indata.get('subfilter'), view = viewList)
-
-
+ viewList = session.subFilter(indata.get('subfilter'), view = viewList)
+
+
dateTo = indata.get('to', int(time.time()))
dateFrom = indata.get('from', dateTo - (86400*30*6)) # Default to a 6 month span
-
+
interval = indata.get('interval', 'month')
-
-
+
+
####################################################################
####################################################################
dOrg = session.user['defaultOrganisation'] or "apache"
@@ -126,7 +126,7 @@ def run(API, environ, indata, session):
if indata.get('email'):
query['query']['bool']['should'] = [{'term': {'sender': indata.get('email')}}]
query['query']['bool']['minimum_should_match'] = 1
-
+
# Get number of committers, this period
query['aggs'] = {
'timeseries': {
@@ -142,7 +142,7 @@ def run(API, environ, indata, session):
size = 0,
body = query
)
-
+
timeseries = []
for bucket in res['aggregations']['timeseries']['buckets']:
ts = int(bucket['key'] / 1000)
@@ -150,7 +150,7 @@ def run(API, environ, indata, session):
'date': ts,
'emails': bucket['doc_count']
})
-
+
JSON_OUT = {
'widgetType': {
'chartType': 'bar' # Recommendation for the UI
diff --git a/api/pages/mail/timeseries.py b/api/pages/mail/timeseries.py
index 7b446f42..ab4e12a1 100644
--- a/api/pages/mail/timeseries.py
+++ b/api/pages/mail/timeseries.py
@@ -56,7 +56,7 @@
# security:
# - cookieAuth: []
# summary: Shows email sent over time
-#
+#
########################################################################
@@ -72,33 +72,33 @@
import hashlib
def run(API, environ, indata, session):
-
+
# We need to be logged in for this!
if not session.user:
raise API.exception(403, "You must be logged in to use this API endpoint! %s")
-
+
now = time.time()
-
+
# First, fetch the view if we have such a thing enabled
viewList = []
if indata.get('view'):
viewList = session.getView(indata.get('view'))
if indata.get('subfilter'):
- viewList = session.subFilter(indata.get('subfilter'), view = viewList)
-
-
+ viewList = session.subFilter(indata.get('subfilter'), view = viewList)
+
+
dateTo = indata.get('to', int(time.time()))
dateFrom = indata.get('from', dateTo - (86400*30*6)) # Default to a 6 month span
-
+
which = 'committer_email'
role = 'committer'
if indata.get('author', False):
which = 'author_email'
role = 'author'
-
+
interval = indata.get('interval', 'month')
-
-
+
+
####################################################################
####################################################################
dOrg = session.user['defaultOrganisation'] or "apache"
@@ -131,7 +131,7 @@ def run(API, environ, indata, session):
if indata.get('email'):
query['query']['bool']['should'] = [{'term': {'sender': indata.get('email')}}]
query['query']['bool']['minimum_should_match'] = 1
-
+
# Get number of committers, this period
query['aggs'] = {
'timeseries': {
@@ -164,7 +164,7 @@ def run(API, environ, indata, session):
size = 0,
body = query
)
-
+
timeseries = []
for bucket in res['aggregations']['timeseries']['buckets']:
ts = int(bucket['key'] / 1000)
@@ -174,7 +174,7 @@ def run(API, environ, indata, session):
'topics': bucket['topics']['value'],
'authors': bucket['authors']['value']
})
-
+
JSON_OUT = {
'widgetType': {
'chartType': 'bar' # Recommendation for the UI
diff --git a/api/pages/mail/top-authors.py b/api/pages/mail/top-authors.py
index 52da07df..c2f2cb21 100644
--- a/api/pages/mail/top-authors.py
+++ b/api/pages/mail/top-authors.py
@@ -56,7 +56,7 @@
# security:
# - cookieAuth: []
# summary: Shows the top N of email authors
-#
+#
########################################################################
@@ -75,27 +75,27 @@
ROBITS = r"(git|jira|jenkins|gerrit)@"
def run(API, environ, indata, session):
-
+
# We need to be logged in for this!
if not session.user:
raise API.exception(403, "You must be logged in to use this API endpoint! %s")
-
+
now = time.time()
-
+
# First, fetch the view if we have such a thing enabled
viewList = []
if indata.get('view'):
viewList = session.getView(indata.get('view'))
if indata.get('subfilter'):
- viewList = session.subFilter(indata.get('subfilter'), view = viewList)
-
-
+ viewList = session.subFilter(indata.get('subfilter'), view = viewList)
+
+
dateTo = indata.get('to', int(time.time()))
dateFrom = indata.get('from', dateTo - (86400*30*6)) # Default to a 6 month span
-
+
interval = indata.get('interval', 'month')
-
-
+
+
####################################################################
####################################################################
dOrg = session.user['defaultOrganisation'] or "apache"
@@ -125,7 +125,7 @@ def run(API, environ, indata, session):
query['query']['bool']['must'].append({'term': {'sourceID': indata.get('source')}})
elif viewList:
query['query']['bool']['must'].append({'terms': {'sourceID': viewList}})
-
+
# Get top 25 committers this period
query['aggs'] = {
'authors': {
@@ -133,7 +133,7 @@ def run(API, environ, indata, session):
'field': 'sender',
'size': 30
}
- }
+ }
}
res = session.DB.ES.search(
index=session.DB.dbname,
@@ -147,7 +147,7 @@ def run(API, environ, indata, session):
email = bucket['key']
# By default, we want to see humans, not bots on this list!
if re.match(ROBITS, email):
- continue
+ continue
count = bucket['doc_count']
sha = hashlib.sha1( ("%s%s" % (dOrg, email)).encode('utf-8') ).hexdigest()
if session.DB.ES.exists(index=session.DB.dbname,doc_type="person",id = sha):
@@ -161,12 +161,12 @@ def run(API, environ, indata, session):
people[email] = person
people[email]['gravatar'] = hashlib.md5(person.get('email', 'unknown').encode('utf-8')).hexdigest()
people[email]['count'] = count
-
+
topN = []
for email, person in people.items():
topN.append(person)
topN = sorted(topN, key = lambda x: x['count'], reverse = True)
-
+
JSON_OUT = {
'topN': {
'denoter': 'emails',
diff --git a/api/pages/mail/top-topics.py b/api/pages/mail/top-topics.py
index 9acda6cf..c9af57cc 100644
--- a/api/pages/mail/top-topics.py
+++ b/api/pages/mail/top-topics.py
@@ -56,7 +56,7 @@
# security:
# - cookieAuth: []
# summary: Shows the top N of email authors
-#
+#
########################################################################
@@ -72,27 +72,27 @@
import hashlib
def run(API, environ, indata, session):
-
+
# We need to be logged in for this!
if not session.user:
raise API.exception(403, "You must be logged in to use this API endpoint! %s")
-
+
now = time.time()
-
+
# First, fetch the view if we have such a thing enabled
viewList = []
if indata.get('view'):
viewList = session.getView(indata.get('view'))
if indata.get('subfilter'):
- viewList = session.subFilter(indata.get('subfilter'), view = viewList)
-
-
+ viewList = session.subFilter(indata.get('subfilter'), view = viewList)
+
+
dateTo = indata.get('to', int(time.time()))
dateFrom = indata.get('from', dateTo - (86400*30*6)) # Default to a 6 month span
-
+
interval = indata.get('interval', 'month')
-
-
+
+
####################################################################
####################################################################
dOrg = session.user['defaultOrganisation'] or "apache"
@@ -125,7 +125,7 @@ def run(API, environ, indata, session):
query['query']['bool']['must'].append({'term': {'sourceID': indata.get('source')}})
elif viewList:
query['query']['bool']['must'].append({'terms': {'sourceID': viewList}})
-
+
res = session.DB.ES.search(
index=session.DB.dbname,
doc_type="mailtop",
@@ -139,8 +139,8 @@ def run(API, environ, indata, session):
'source': bucket['_source']['sourceURL'],
'name': bucket['_source']['subject'],
'count': bucket['_source']['emails']
- })
-
+ })
+
JSON_OUT = {
'topN': {
'denoter': 'emails',
diff --git a/api/pages/mail/trends.py b/api/pages/mail/trends.py
index ac0d5186..baa1f1d4 100644
--- a/api/pages/mail/trends.py
+++ b/api/pages/mail/trends.py
@@ -56,7 +56,7 @@
# security:
# - cookieAuth: []
# summary: Shows a quick email trend summary of the past 6 months for your org
-#
+#
########################################################################
@@ -72,30 +72,30 @@
import datetime
def run(API, environ, indata, session):
-
+
# We need to be logged in for this!
if not session.user:
raise API.exception(403, "You must be logged in to use this API endpoint! %s")
-
+
now = time.time()
-
+
# First, fetch the view if we have such a thing enabled
viewList = []
if indata.get('view'):
viewList = session.getView(indata.get('view'))
if indata.get('subfilter'):
- viewList = session.subFilter(indata.get('subfilter'), view = viewList)
-
-
+ viewList = session.subFilter(indata.get('subfilter'), view = viewList)
+
+
dateTo = indata.get('to', int(time.time()))
dateFrom = indata.get('from', dateTo - (86400*30*6)) # Default to a 6 month span
if dateFrom < 0:
dateFrom = 0
dateYonder = dateFrom - (dateTo - dateFrom)
-
-
+
+
dOrg = session.user['defaultOrganisation'] or "apache"
-
+
####################################################################
# We start by doing all the queries for THIS period. #
# Then we reset the query, and change date to yonder-->from #
@@ -129,8 +129,8 @@ def run(API, environ, indata, session):
query['query']['bool']['must'].append({'terms': {'sourceID': viewList}})
if indata.get('email'):
query['query']['bool']['must'].append({'term': {'sender': indata.get('email')}})
-
-
+
+
# Get number of threads and emails, this period
query['aggs'] = {
'topics': {
@@ -152,10 +152,10 @@ def run(API, environ, indata, session):
)
no_topics = res['aggregations']['topics']['value']
no_emails = res['aggregations']['emails']['value']
-
-
+
+
# Authors
-
+
query = {
'query': {
'bool': {
@@ -184,7 +184,7 @@ def run(API, environ, indata, session):
query['query']['bool']['must'].append({'terms': {'sourceID': viewList}})
if indata.get('email'):
query['query']['bool']['must'].append({'term': {'sender': indata.get('email')}})
-
+
# Get number of authors, this period
query['aggs'] = {
'authors': {
@@ -200,9 +200,9 @@ def run(API, environ, indata, session):
body = query
)
no_authors = res['aggregations']['authors']['value']
-
-
-
+
+
+
####################################################################
# Change to PRIOR SPAN #
####################################################################
@@ -234,8 +234,8 @@ def run(API, environ, indata, session):
query['query']['bool']['must'].append({'terms': {'sourceID': viewList}})
if indata.get('email'):
query['query']['bool']['must'].append({'term': {'sender': indata.get('email')}})
-
-
+
+
# Get number of threads and emails, this period
query['aggs'] = {
'topics': {
@@ -257,10 +257,10 @@ def run(API, environ, indata, session):
)
no_topics_before = res['aggregations']['topics']['value']
no_emails_before = res['aggregations']['emails']['value']
-
-
+
+
# Authors
-
+
query = {
'query': {
'bool': {
@@ -289,7 +289,7 @@ def run(API, environ, indata, session):
query['query']['bool']['must'].append({'terms': {'sourceID': viewList}})
if indata.get('email'):
query['query']['bool']['must'].append({'term': {'sender': indata.get('email')}})
-
+
# Get number of authors, this period
query['aggs'] = {
'authors': {
@@ -305,10 +305,10 @@ def run(API, environ, indata, session):
body = query
)
no_authors_before = res['aggregations']['authors']['value']
-
-
-
-
+
+
+
+
trends = {
"authors": {
'before': no_authors_before,
@@ -326,7 +326,7 @@ def run(API, environ, indata, session):
'title': "Emails sent this period"
}
}
-
+
JSON_OUT = {
'trends': trends,
'okay': True,
diff --git a/api/pages/org/contributors.py b/api/pages/org/contributors.py
index 9210dfff..3d243622 100644
--- a/api/pages/org/contributors.py
+++ b/api/pages/org/contributors.py
@@ -39,7 +39,7 @@
# security:
# - cookieAuth: []
# summary: Shows contributors for the entire org or matching filters.
-#
+#
########################################################################
@@ -57,20 +57,20 @@
cached_people = {} # Store people we know, so we don't have to fetch them again.
def run(API, environ, indata, session):
-
+
# We need to be logged in for this!
if not session.user:
raise API.exception(403, "You must be logged in to use this API endpoint! %s")
-
-
+
+
# First, fetch the view if we have such a thing enabled
viewList = []
if indata.get('view'):
viewList = session.getView(indata.get('view'))
if indata.get('subfilter'):
- viewList = session.subFilter(indata.get('subfilter'), view = viewList)
-
-
+ viewList = session.subFilter(indata.get('subfilter'), view = viewList)
+
+
# Fetch all contributors for the org
dOrg = session.user['defaultOrganisation'] or "apache"
query = {
@@ -86,13 +86,13 @@ def run(API, environ, indata, session):
}
}
}
-
+
# Source-specific or view-specific??
if indata.get('source'):
query['query']['bool']['must'].append({'term': {'sourceID': indata.get('source')}})
elif viewList:
query['query']['bool']['must'].append({'terms': {'sourceID': viewList}})
-
+
# Date specific?
dateTo = indata.get('to', int(time.time()))
dateFrom = indata.get('from', dateTo - (86400*30*6)) # Default to a 6 month span
@@ -108,7 +108,7 @@ def run(API, environ, indata, session):
)
emails = []
contribs = {}
-
+
for field in ['sender', 'author_email', 'issueCreator', 'issueCloser']:
N = 0
while N < 5:
@@ -121,7 +121,7 @@ def run(API, environ, indata, session):
'partition': N,
'num_partitions': 5
},
- }
+ }
}
}
res = session.DB.ES.search(
@@ -139,7 +139,7 @@ def run(API, environ, indata, session):
emails.append(k['key'])
contribs[k['key']] = contribs.get(k['key'], 0) + k['doc_count']
N += 1
-
+
people = []
for email in emails:
pid = hashlib.sha1( ("%s%s" % (dOrg, email)).encode('ascii', errors='replace')).hexdigest()
@@ -160,7 +160,7 @@ def run(API, environ, indata, session):
if person:
person['contributions'] = contribs.get(email, 0)
people.append(person)
-
+
JSON_OUT = {
'people': people,
'okay': True
diff --git a/api/pages/org/list.py b/api/pages/org/list.py
index 28d6d6f5..bf28f4d6 100644
--- a/api/pages/org/list.py
+++ b/api/pages/org/list.py
@@ -83,7 +83,7 @@
# security:
# - cookieAuth: []
# summary: Create a new organisation
-#
+#
########################################################################
@@ -102,7 +102,7 @@ def run(API, environ, indata, session):
# We need to be logged in for this!
if not session.user:
raise API.exception(403, "You must be logged in to use this API endpoint!")
-
+
method = environ['REQUEST_METHOD']
# Are we making a new org?
if method == "PUT":
@@ -112,7 +112,7 @@ def run(API, environ, indata, session):
orgid = indata.get('id', str(int(time.time())))
if session.DB.ES.exists(index=session.DB.dbname, doc_type='organisation', id = orgid):
raise API.exception(403, "Organisation ID already in use!")
-
+
doc = {
'id': orgid,
'name': orgname,
@@ -125,7 +125,7 @@ def run(API, environ, indata, session):
return
else:
raise API.exception(403, "Only administrators can create new organisations.")
-
+
####################################################################
orgs = []
if session.user['userlevel'] == "admin":
@@ -168,8 +168,8 @@ def run(API, environ, indata, session):
doc['_source']['sourceCount'] = numSources
doc['_source']['docCount'] = numDocs
orgs.append(doc['_source'])
-
-
+
+
JSON_OUT = {
'organisations': orgs,
'okay': True,
diff --git a/api/pages/org/members.py b/api/pages/org/members.py
index 3b58852c..b749b2fd 100644
--- a/api/pages/org/members.py
+++ b/api/pages/org/members.py
@@ -106,7 +106,7 @@
# security:
# - cookieAuth: []
# summary: Remove a person from an organisation
-#
+#
########################################################################
@@ -125,7 +125,7 @@ def canInvite(session):
""" Determine if the user can edit sources in this org """
if session.user['userlevel'] == 'admin':
return True
-
+
dOrg = session.user['defaultOrganisation'] or "apache"
if session.DB.ES.exists(index=session.DB.dbname, doc_type="organisation", id= dOrg):
xorg = session.DB.ES.get(index=session.DB.dbname, doc_type="organisation", id= dOrg)['_source']
@@ -138,9 +138,9 @@ def run(API, environ, indata, session):
# We need to be logged in for this!
if not session.user:
raise API.exception(403, "You must be logged in to use this API endpoint!")
-
+
method = environ['REQUEST_METHOD']
-
+
#################################################
# Inviting a new member? #
#################################################
@@ -152,18 +152,18 @@ def run(API, environ, indata, session):
# Make sure the org exists
if not session.DB.ES.exists(index=session.DB.dbname, doc_type='organisation', id = orgid):
raise API.exception(403, "No such organisation!")
-
+
# make sure the user account exists
if not session.DB.ES.exists(index=session.DB.dbname, doc_type='useraccount', id = newmember):
raise API.exception(403, "No such user!")
-
+
# Modify user account
doc = session.DB.ES.get(index=session.DB.dbname, doc_type='useraccount', id = newmember)
if orgid not in doc['_source']['organisations']: # No duplicates, please
doc['_source']['organisations'].append(orgid)
session.DB.ES.index(index=session.DB.dbname, doc_type='useraccount', id = newmember, body = doc['_source'])
-
-
+
+
# Get org doc from ES
doc = session.DB.ES.get(index=session.DB.dbname, doc_type='organisation', id = orgid)
if isadmin:
@@ -172,7 +172,7 @@ def run(API, environ, indata, session):
# Override old doc
session.DB.ES.index(index=session.DB.dbname, doc_type='organisation', id = orgid, body = doc['_source'])
time.sleep(1) # Bleh!!
-
+
# If an admin, and not us, and reinvited, we purge the admin bit
elif newmember in doc['_source']['admins']:
if newmember == session.user['email']:
@@ -182,11 +182,11 @@ def run(API, environ, indata, session):
session.DB.ES.index(index=session.DB.dbname, doc_type='organisation', id = orgid, body = doc['_source'])
time.sleep(1) # Bleh!!
yield json.dumps({"okay": True, "message": "Member invited!!"})
-
+
return
else:
raise API.exception(403, "Only administrators or organisation owners can invite new members.")
-
+
#################################################
# DELETE: Remove a member #
#################################################
@@ -195,25 +195,25 @@ def run(API, environ, indata, session):
memberid = indata.get('email')
isadmin = indata.get('admin', False)
orgid = session.user['defaultOrganisation'] or "apache"
-
+
# We can't remove ourselves!
if memberid == session.user['email']:
raise API.exception(403, "You can't remove yourself from an organisation.")
-
+
# Make sure the org exists
if not session.DB.ES.exists(index=session.DB.dbname, doc_type='organisation', id = orgid):
raise API.exception(403, "No such organisation!")
-
+
# make sure the user account exists
if not session.DB.ES.exists(index=session.DB.dbname, doc_type='useraccount', id = memberid):
raise API.exception(403, "No such user!")
-
+
# Modify user account
doc = session.DB.ES.get(index=session.DB.dbname, doc_type='useraccount', id = memberid)
if orgid in doc['_source']['organisations']: # No duplicates, please
doc['_source']['organisations'].remove(orgid)
session.DB.ES.index(index=session.DB.dbname, doc_type='useraccount', id = memberid, body = doc['_source'])
-
+
# Check is user is admin and remove if so
# Get org doc from ES
doc = session.DB.ES.get(index=session.DB.dbname, doc_type='organisation', id = orgid)
@@ -222,12 +222,12 @@ def run(API, environ, indata, session):
# Override old doc
session.DB.ES.index(index=session.DB.dbname, doc_type='organisation', id = orgid, body = doc['_source'])
time.sleep(1) # Bleh!!
-
+
yield json.dumps({"okay": True, "message": "Member removed!"})
return
else:
raise API.exception(403, "Only administrators or organisation owners can invite new members.")
-
+
#################################################
# GET/POST: Display members #
@@ -236,11 +236,11 @@ def run(API, environ, indata, session):
orgid = session.user['defaultOrganisation'] or "apache"
if not session.DB.ES.exists(index=session.DB.dbname, doc_type='organisation', id = orgid):
raise API.exception(403, "No such organisation!")
-
+
# Only admins should be able to view this!
if not canInvite(session):
raise API.exception(403, "Only organisation owners can view this list.")
-
+
# Find everyone affiliated with this org
query = {
'query': {
@@ -264,7 +264,7 @@ def run(API, environ, indata, session):
members = []
for doc in res['hits']['hits']:
members.append(doc['_id'])
-
+
# Get org doc from ES
doc = session.DB.ES.get(index=session.DB.dbname, doc_type='organisation', id = orgid)
JSON_OUT = {
diff --git a/api/pages/org/sourcetypes.py b/api/pages/org/sourcetypes.py
index 9e5b8afe..6fa84025 100644
--- a/api/pages/org/sourcetypes.py
+++ b/api/pages/org/sourcetypes.py
@@ -56,7 +56,7 @@
# security:
# - cookieAuth: []
# summary: Lists the available source types supported by Kibble
-#
+#
########################################################################
@@ -71,9 +71,7 @@
import json
def run(API, environ, indata, session):
-
+
types = yaml.load(open("yaml/sourcetypes.yaml"))
-
- yield json.dumps(types)
-
\ No newline at end of file
+ yield json.dumps(types)
diff --git a/api/pages/org/trends.py b/api/pages/org/trends.py
index d0188a41..4890c63f 100644
--- a/api/pages/org/trends.py
+++ b/api/pages/org/trends.py
@@ -56,7 +56,7 @@
# security:
# - cookieAuth: []
# summary: Shows a quick trend summary of the past 6 months for your org
-#
+#
########################################################################
@@ -71,28 +71,28 @@
import time
def run(API, environ, indata, session):
-
+
# We need to be logged in for this!
if not session.user:
raise API.exception(403, "You must be logged in to use this API endpoint! %s")
-
+
now = time.time()
-
+
# First, fetch the view if we have such a thing enabled
viewList = []
if indata.get('view'):
if session.DB.ES.exists(index=session.DB.dbname, doc_type="view", id = indata['view']):
view = session.DB.ES.get(index=session.DB.dbname, doc_type="view", id = indata['view'])
viewList = view['_source']['sourceList']
-
+
dateTo = int(time.time())
dateFrom = dateTo - (86400*30*3) # Default to a quarter
if dateFrom < 0:
dateFrom = 0
dateYonder = dateFrom - (dateTo - dateFrom)
-
-
-
+
+
+
####################################################################
# We start by doing all the queries for THIS period. #
# Then we reset the query, and change date to yonder-->from #
@@ -120,7 +120,7 @@ def run(API, environ, indata, session):
}
}
}
-
+
# Get number of commits, this period
res = session.DB.ES.count(
index=session.DB.dbname,
@@ -128,8 +128,8 @@ def run(API, environ, indata, session):
body = query
)
no_commits = res['count']
-
-
+
+
# Get number of committers, this period
query['aggs'] = {
'authors': {
@@ -137,7 +137,7 @@ def run(API, environ, indata, session):
'field': 'author_email'
}
}
-
+
}
res = session.DB.ES.search(
index=session.DB.dbname,
@@ -146,8 +146,8 @@ def run(API, environ, indata, session):
body = query
)
no_authors = res['aggregations']['authors']['value']
-
-
+
+
####################################################################
# Change to PRIOR SPAN #
####################################################################
@@ -173,7 +173,7 @@ def run(API, environ, indata, session):
}
}
}
-
+
# Get number of commits, this period
res = session.DB.ES.count(
index=session.DB.dbname,
@@ -181,7 +181,7 @@ def run(API, environ, indata, session):
body = query
)
no_commits_before = res['count']
-
+
# Get number of committers, this period
query['aggs'] = {
'authors': {
@@ -197,8 +197,8 @@ def run(API, environ, indata, session):
body = query
)
no_authors_before = res['aggregations']['authors']['value']
-
-
+
+
trends = {
"authors": {
'before': no_authors_before,
@@ -211,7 +211,7 @@ def run(API, environ, indata, session):
'title': "Commits this quarter"
}
}
-
+
JSON_OUT = {
'trends': trends,
'okay': True,
diff --git a/api/pages/session.py b/api/pages/session.py
index 425cd89a..d09daabe 100644
--- a/api/pages/session.py
+++ b/api/pages/session.py
@@ -85,7 +85,7 @@
# $ref: '#/components/schemas/Error'
# description: unexpected error
# summary: Log in
-#
+#
########################################################################
@@ -104,9 +104,9 @@
import uuid
def run(API, environ, indata, session):
-
+
method = environ['REQUEST_METHOD']
-
+
# Logging in?
if method == "PUT":
u = indata['email']
@@ -127,30 +127,30 @@ def run(API, environ, indata, session):
session.DB.ES.index(index=session.DB.dbname, doc_type='uisession', id = session.cookie, body = sessionDoc)
yield json.dumps({"message": "Logged in OK!"})
return
-
+
# Fall back to a 403 if username and password did not match
raise API.exception(403, "Wrong username or password supplied!")
-
-
+
+
# We need to be logged in for the rest of this!
if not session.user:
raise API.exception(403, "You must be logged in to use this API endpoint! %s")
-
+
# Delete a session (log out)
if method == "DELETE":
session.DB.ES.delete(index=session.DB.dbname, doc_type='uisession', id = session.cookie)
session.newCookie()
yield json.dumps({"message": "Logged out, bye bye!"})
-
+
# Display the user data for this session
if method == "GET":
-
+
# Do we have an API key? If not, make one
if not session.user.get('token') or indata.get('newtoken'):
token = str(uuid.uuid4())
session.user['token'] = token
session.DB.ES.index(index=session.DB.dbname, doc_type='useraccount', id = session.user['email'], body = session.user)
-
+
# Run a quick search of all orgs we have.
res = session.DB.ES.search(
index=session.DB.dbname,
@@ -162,12 +162,12 @@ def run(API, environ, indata, session):
}
}
)
-
+
orgs = []
for hit in res['hits']['hits']:
doc = hit['_source']
orgs.append(doc)
-
+
JSON_OUT = {
'email': session.user['email'],
'displayName': session.user['displayName'],
@@ -180,7 +180,6 @@ def run(API, environ, indata, session):
}
yield json.dumps(JSON_OUT)
return
-
+
# Finally, if we hit a method we don't know, balk!
yield API.exception(400, "I don't know this request method!!")
-
\ No newline at end of file
diff --git a/api/pages/sources.py b/api/pages/sources.py
index 0a46756b..a15a55b0 100644
--- a/api/pages/sources.py
+++ b/api/pages/sources.py
@@ -114,7 +114,7 @@
# security:
# - cookieAuth: []
# summary: Add a new source
-#
+#
########################################################################
@@ -133,7 +133,7 @@
def canModifySource(session):
""" Determine if the user can edit sources in this org """
-
+
dOrg = session.user['defaultOrganisation'] or "apache"
if session.DB.ES.exists(index=session.DB.dbname, doc_type="organisation", id= dOrg):
xorg = session.DB.ES.get(index=session.DB.dbname, doc_type="organisation", id= dOrg)['_source']
@@ -144,30 +144,30 @@ def canModifySource(session):
return False
def run(API, environ, indata, session):
-
+
# We need to be logged in for this!
if not session.user:
raise API.exception(403, "You must be logged in to use this API endpoint! %s")
-
+
method = environ['REQUEST_METHOD']
dOrg = session.user['defaultOrganisation']
-
+
if method in ['GET', 'POST']:
# Fetch organisation data
-
+
# Make sure we have a default/current org set
if 'defaultOrganisation' not in session.user or not session.user['defaultOrganisation']:
raise API.exception(400, "You must specify an organisation as default/current in order to add sources.")
-
+
if session.DB.ES.exists(index=session.DB.dbname, doc_type="organisation", id= dOrg):
org = session.DB.ES.get(index=session.DB.dbname, doc_type="organisation", id= dOrg)['_source']
del org['admins']
else:
raise API.exception(404, "No such organisation, '%s'" % (dOrg or "(None)"))
-
+
sourceTypes = indata.get('types', [])
# Fetch all sources for default org
-
+
res = session.DB.ES.search(
index=session.DB.dbname,
doc_type="source",
@@ -180,15 +180,15 @@ def run(API, environ, indata, session):
}
}
)
-
+
# Secondly, fetch the view if we have such a thing enabled
viewList = []
if indata.get('view'):
viewList = session.getView(indata.get('view'))
if indata.get('subfilter') and indata.get('quick'):
- viewList = session.subFilter(indata.get('subfilter'), view = viewList)
-
-
+ viewList = session.subFilter(indata.get('subfilter'), view = viewList)
+
+
sources = []
for hit in res['hits']['hits']:
doc = hit['_source']
@@ -208,7 +208,7 @@ def run(API, environ, indata, session):
if 'creds' in doc:
del doc['creds']
sources.append(doc)
-
+
JSON_OUT = {
'sources': sources,
'okay': True,
@@ -216,7 +216,7 @@ def run(API, environ, indata, session):
}
yield json.dumps(JSON_OUT)
return
-
+
# Add one or more sources
if method == "PUT":
if canModifySource(session):
@@ -234,11 +234,11 @@ def run(API, environ, indata, session):
if el in source and len(source[el]) > 0:
creds[el] = source[el]
sourceID = hashlib.sha224( ("%s-%s" % (sourceType, sourceURL)).encode('utf-8') ).hexdigest()
-
+
# Make sure we have a default/current org set
if 'defaultOrganisation' not in session.user or not session.user['defaultOrganisation']:
raise API.exception(400, "You must first specify an organisation as default/current in order to add sources.")
-
+
doc = {
'organisation': dOrg,
'sourceURL': sourceURL,
@@ -259,7 +259,7 @@ def run(API, environ, indata, session):
})
else:
raise API.exception(403, "You don't have permission to add sources to this organisation.")
-
+
# Delete a source
if method == "DELETE":
if canModifySource(session):
@@ -277,7 +277,7 @@ def run(API, environ, indata, session):
raise API.exception(404, "No such source item")
else:
raise API.exception(403, "You don't have permission to delete this source.")
-
+
# Edit a source
if method == "PATCH":
pass
diff --git a/api/pages/verify.py b/api/pages/verify.py
index 0b4d7071..8ef6f4f1 100644
--- a/api/pages/verify.py
+++ b/api/pages/verify.py
@@ -45,7 +45,7 @@
# application/json:
# schema:
# $ref: '#/components/schemas/Error'
-#
+#
########################################################################
@@ -58,17 +58,17 @@
def run(API, environ, indata, session):
-
+
# Get vocde, make sure it's 40 chars
vcode = indata.get('vcode')
if len(vcode) != 40:
raise API.exception(400, "Invalid verification code!")
-
+
# Find the account with this vcode
email = indata.get('email')
if len(email) < 7:
raise API.exception(400, "Invalid email address presented.")
-
+
if session.DB.ES.exists(index=session.DB.dbname, doc_type='useraccount', id = email):
doc = session.DB.ES.get(index=session.DB.dbname, doc_type='useraccount', id = email)
# Do the codes match??
@@ -81,4 +81,3 @@ def run(API, environ, indata, session):
raise API.exception(404, "Invalid verification code presented!")
else:
raise API.exception(404, "Invalid verification code presented!") # Don't give away if such a user exists, pssst
-
\ No newline at end of file
diff --git a/api/pages/views.py b/api/pages/views.py
index 5898e110..bc619f21 100644
--- a/api/pages/views.py
+++ b/api/pages/views.py
@@ -128,7 +128,7 @@
# security:
# - cookieAuth: []
# summary: Add a new view
-#
+#
########################################################################
@@ -145,14 +145,14 @@
import hashlib
def run(API, environ, indata, session):
-
+
# We need to be logged in for this!
if not session.user:
raise API.exception(403, "You must be logged in to use this API endpoint! %s")
-
+
method = environ['REQUEST_METHOD']
dOrg = session.user['defaultOrganisation'] or "apache"
-
+
# Are we adding a view?
if method == 'PUT':
viewID = hashlib.sha224( ("%s-%s-%s" % (time.time(), session.user['email'], dOrg) ).encode('utf-8') ).hexdigest()
@@ -173,7 +173,7 @@ def run(API, environ, indata, session):
}
session.DB.ES.index(index=session.DB.dbname, doc_type="view", id = viewID, body = doc)
yield json.dumps({'okay': True, 'message': "View created"})
-
+
# Are we editing (patching) a view?
if method == 'PATCH':
viewID = indata.get('id')
@@ -188,7 +188,7 @@ def run(API, environ, indata, session):
raise API.exception(403, "You don't own this view, and cannot edit it.")
else:
raise API.exception(404, "We couldn't find a view with this ID.")
-
+
# Removing a view?
if method == 'DELETE':
viewID = indata.get('id')
@@ -201,11 +201,11 @@ def run(API, environ, indata, session):
raise API.exception(403, "You don't own this view, and cannot delete it.")
else:
raise API.exception(404, "We couldn't find a view with this ID.")
-
-
+
+
if method in ['GET', 'POST']:
# Fetch all views for default org
-
+
res = session.DB.ES.search(
index=session.DB.dbname,
doc_type="view",
@@ -218,8 +218,8 @@ def run(API, environ, indata, session):
}
}
)
-
-
+
+
# Are we looking at someone elses view?
if indata.get('view'):
viewID = indata.get('view')
@@ -229,7 +229,7 @@ def run(API, environ, indata, session):
blob['_source']['name'] += " (shared by " + blob['_source']['email'] + ")"
res['hits']['hits'].append(blob)
sources = []
-
+
# Include public views??
if not indata.get('sources', False):
pres = session.DB.ES.search(
@@ -259,7 +259,7 @@ def run(API, environ, indata, session):
if hit['_source']['email'] != session.user['email']:
hit['_source']['name'] += " (shared view)"
res['hits']['hits'].append(hit)
-
+
for hit in res['hits']['hits']:
doc = hit['_source']
if doc['organisation'] != dOrg:
@@ -273,7 +273,7 @@ def run(API, environ, indata, session):
sources.append(xdoc)
else:
sources.append(doc)
-
+
allsources = []
if indata.get('sources', False):
res = session.DB.ES.search(
@@ -296,7 +296,7 @@ def run(API, environ, indata, session):
'sourceURL': doc['sourceURL']
}
allsources.append(xdoc)
-
+
JSON_OUT = {
'views': sources,
'sources': allsources,
diff --git a/api/pages/widgets.py b/api/pages/widgets.py
index cebda02a..df5d4389 100644
--- a/api/pages/widgets.py
+++ b/api/pages/widgets.py
@@ -50,12 +50,12 @@
import json
def run(API, environ, indata, session):
-
+
if not session.user:
raise API.exception(403, "You must be logged in to use this API endpoint! %s")
-
+
widgets = yaml.load(open("yaml/widgets.yaml"))
-
+
page = indata['pageid']
if not page or page == '0':
page = widgets.get('defaultWidget', 'repos')
@@ -63,5 +63,3 @@ def run(API, environ, indata, session):
yield json.dumps(widgets['widgets'][page])
else:
raise API.exception(404, "Widget design not found!")
-
-
\ No newline at end of file
diff --git a/api/plugins/database.py b/api/plugins/database.py
index 80b94dd1..395808d4 100644
--- a/api/plugins/database.py
+++ b/api/plugins/database.py
@@ -34,7 +34,7 @@ class KibbleESWrapper(object):
"""
def __init__(self, ES):
self.ES = ES
-
+
def get(self, index, doc_type, id):
return self.ES.get(index = index+'_'+doc_type, doc_type = '_doc', id = id)
def exists(self, index, doc_type, id):
@@ -72,7 +72,7 @@ class KibbleESWrapperSeven(object):
"""
def __init__(self, ES):
self.ES = ES
-
+
def get(self, index, doc_type, id):
return self.ES.get(index = index+'_'+doc_type, id = id)
def exists(self, index, doc_type, id):
@@ -100,7 +100,7 @@ def count(self, index, doc_type = '*', body = None):
index = index+'_'+doc_type,
body = body
)
-
+
class KibbleDatabase(object):
def __init__(self, config):
@@ -117,7 +117,7 @@ def __init__(self, config):
max_retries=5,
retry_on_timeout=True
)
-
+
# IMPORTANT BIT: Figure out if this is ES < 6.x, 6.x or >= 7.x.
# If so, we're using the new ES DB mappings, and need to adjust ALL
# ES calls to match this.
@@ -126,4 +126,3 @@ def __init__(self, config):
self.ES = KibbleESWrapperSeven(self.ES)
elif self.ESVersion >= 6:
self.ES = KibbleESWrapper(self.ES)
-
diff --git a/api/plugins/openapi.py b/api/plugins/openapi.py
index ba6153a5..f2a7e3f0 100644
--- a/api/plugins/openapi.py
+++ b/api/plugins/openapi.py
@@ -48,7 +48,7 @@ def __init__(self, message):
'POST': '#49cc5c',
'PATCH': '#d5a37e'
}
-
+
class OpenAPI():
def __init__(self, APIFile):
""" Instantiates an OpenAPI validator given a YAML specification"""
@@ -56,26 +56,26 @@ def __init__(self, APIFile):
self.API = json.load(open(APIFile))
else:
self.API = yaml.load(open(APIFile))
-
+
def validateType(self, field, value, ftype):
""" Validate a single field value against an expected type """
-
+
# Get type of value, convert to JSON name of type.
pyType = type(value).__name__
jsonType = py2JSON[pyType] if pyType in py2JSON else pyType
-
+
# Check if type matches
if ftype != jsonType:
raise OpenAPIException("OpenAPI mismatch: Field '%s' was expected to be %s, but was really %s!" % (field, ftype, jsonType))
-
+
def validateSchema(self, pdef, formdata, schema = None):
""" Validate (sub)parameters against OpenAPI specs """
-
+
# allOf: list of schemas to validate against
if 'allOf' in pdef:
for subdef in pdef['allOf']:
self.validateSchema(subdef, formdata)
-
+
where = "JSON body"
# Symbolic link??
if 'schema' in pdef:
@@ -86,13 +86,13 @@ def validateSchema(self, pdef, formdata, schema = None):
# #/foo/bar/baz --> dict['foo']['bar']['baz']
pdef = functools.reduce(operator.getitem, schema.split('/')[1:], self.API)
where = "item matching schema %s" % schema
-
+
# Check that all required fields are present
if 'required' in pdef:
for field in pdef['required']:
if not field in formdata:
raise OpenAPIException("OpenAPI mismatch: Missing input field '%s' in %s!" % (field, where))
-
+
# Now check for valid format of input data
for field in formdata:
if 'properties' not in pdef or field not in pdef['properties'] :
@@ -101,7 +101,7 @@ def validateSchema(self, pdef, formdata, schema = None):
raise OpenAPIException("OpenAPI mismatch: Field '%s' was found in api.yaml, but no format was specified in specs!" % field)
ftype = pdef['properties'][field]['type']
self.validateType(field, formdata[field], ftype)
-
+
# Validate sub-arrays
if ftype == 'array' and 'items' in pdef['properties'][field]:
for item in formdata[field]:
@@ -109,17 +109,17 @@ def validateSchema(self, pdef, formdata, schema = None):
self.validateSchema(pdef['properties'][field]['items'], item)
else:
self.validateType(field, formdata[field], pdef['properties'][field]['items']['type'])
-
+
# Validate sub-hashes
if ftype == 'hash' and 'schema' in pdef['properties'][field]:
self.validateSchema(pdef['properties'][field], formdata[field])
def validateParameters(self, defs, formdata):
#
pass
-
+
def validate(self, method = "GET", path = "/foo", formdata = None):
""" Validate the request method and input data against the OpenAPI specification """
-
+
# Make sure we're not dealing with a dynamic URL.
# If we find /foo/{key}, we fold that into the form data
# and process as if it's a json input field for now.
@@ -132,7 +132,7 @@ def validate(self, method = "GET", path = "/foo", formdata = None):
formdata[k] = v
path = xpath
break
-
+
if self.API['paths'].get(path):
defs = self.API['paths'].get(path)
method = method.lower()
@@ -143,15 +143,15 @@ def validate(self, method = "GET", path = "/foo", formdata = None):
elif formdata and 'requestBody' not in mdefs:
raise OpenAPIException("OpenAPI mismatch: JSON data is now allowed for this request type")
elif formdata and 'requestBody' in mdefs and 'content' in mdefs['requestBody']:
-
+
# SHORTCUT: We only care about JSON input for Kibble! Disregard other types
if not 'application/json' in mdefs['requestBody']['content']:
raise OpenAPIException ("OpenAPI mismatch: API endpoint accepts input, but no application/json definitions found in api.yaml!")
jdefs = mdefs['requestBody']['content']['application/json']
-
+
# Check that required params are here
self.validateSchema(jdefs, formdata)
-
+
else:
raise OpenAPIException ("OpenAPI mismatch: Method %s is not registered for this API" % method)
else:
@@ -184,7 +184,7 @@ def dumpExamples(self, pdef, array = False):
else:
js[k], foo = self.dumpExamples(v['items'])
return [js if not array else [js], desc]
-
+
def toHTML(self):
""" Blurps out the specs in a pretty HTML blob """
print("""
@@ -217,7 +217,7 @@ def toHTML(self):
xjs, desc = self.dumpExamples(pdef)
js = json.dumps(xjs, indent = 4)
resp += "\n" % (code, js)
-
+
if 'requestBody' in mspec:
for ctype, pdef in mspec['requestBody']['content'].items():
xjs, desc = self.dumpExamples(pdef)
@@ -226,18 +226,18 @@ def toHTML(self):
inpvars += "%s: (%s) %s
\n" % (k, v[0], v[1])
js = json.dumps(xjs, indent = 4)
inp += "" % (ctype, js)
-
+
if inpvars:
inpvars = "" % inpvars
-
+
print("""
-
+
%s
-
+
%s
@@ -256,4 +256,4 @@ def toHTML(self):
""" % (linkname, mcolors[method], mcolors[method], mcolors[method], method, path, summary, "block" if inp else "none", inpvars, inp, resp))
#print("%s %s: %s" % (method.upper(), path, mspec['summary']))
- print("