452 lines
19 KiB
Python
452 lines
19 KiB
Python
import settings,os
|
|
import ng_openstack.openstackRequest
|
|
import exceptions
|
|
import json
|
|
import pybill_redis
|
|
import requests
|
|
import datetime, pytz, calendar
|
|
|
|
appDebug = True
|
|
|
|
|
|
myAccountingRedisCon=pybill_redis.pyRedis(os.getenv("REDIS_HOST"),6379,"")
|
|
PREFIX="pyBill_"
|
|
|
|
|
|
|
|
|
|
def add_months(date, months):
|
|
months_count = date.month + months
|
|
|
|
# Calculate the year
|
|
year = date.year + int(months_count / 12)
|
|
|
|
# Calculate the month
|
|
month = (months_count % 12)
|
|
if month == 0:
|
|
month = 12
|
|
|
|
# Calculate the day
|
|
day = date.day
|
|
last_day_of_month = calendar.monthrange(year, month)[1]
|
|
if day > last_day_of_month:
|
|
day = last_day_of_month
|
|
|
|
new_date = datetime.date(year, month, day)
|
|
return new_date
|
|
|
|
def getBWUsageByMonthforProject(yearNumber, monthNumber, projectID):
|
|
fmt = '%Y-%m-%dT%H:%M:%S'
|
|
# YY,MM,DD
|
|
reportingMonth = datetime.datetime(int(yearNumber), int(monthNumber), 1)
|
|
|
|
midnightOnFirstDayofMonthInUTC = datetime.datetime.now(pytz.timezone("UTC")) \
|
|
.replace(year=reportingMonth.year, month=reportingMonth.month, day=1, hour=0, minute=0, second=0, microsecond=0) \
|
|
.astimezone(pytz.utc)
|
|
|
|
startTime = midnightOnFirstDayofMonthInUTC.strftime(fmt)
|
|
|
|
|
|
# Should we roll to next year?
|
|
theNextMonth = add_months(reportingMonth, 1).month
|
|
theNextYear = add_months(reportingMonth, 1).year
|
|
|
|
midnightOnFirstDayofFollowingMonthInUTC = datetime.datetime.now(pytz.timezone("UTC")) \
|
|
.replace(year=theNextYear, month=theNextMonth, day=1, hour=0, minute=0, second=0,
|
|
microsecond=0) \
|
|
.astimezone(pytz.utc)
|
|
|
|
endTime = midnightOnFirstDayofFollowingMonthInUTC.strftime(fmt)
|
|
|
|
return getUsageDataByProject(projectID,startTime,endTime)
|
|
|
|
def getUsageDataByProjectandMetricType(projectID,startTime,endTime,metricType):
|
|
|
|
|
|
data = {
|
|
"operations": [
|
|
"aggregate",
|
|
"sum",
|
|
[
|
|
"metric",
|
|
metricType,
|
|
"sum"
|
|
]
|
|
],
|
|
"resource_type": "heliostack_bandwidth_meter",
|
|
"search": {
|
|
"=": {
|
|
"project_id":projectID
|
|
}
|
|
}
|
|
}
|
|
|
|
try:
|
|
usageData = ng_openstack.openstackRequest.openstackRequest("POST", "aggregates?granularity=3600&start="+startTime+"&stop="+endTime,data,
|
|
'http://' + os.getenv("GNOCCHI_HOST") + ':8041/v1')
|
|
# print(usageData.json())
|
|
# print(str(len(usageData.json()['measures']['aggregated'])) + " Entries")
|
|
sumTotal = 0
|
|
for thisMeasure in usageData.json()['measures']['aggregated']:
|
|
if thisMeasure[2]>1000000000000:
|
|
thisMeasure[2]=0
|
|
if thisMeasure[2]<0:
|
|
thisMeasure[2]=0
|
|
sumTotal += thisMeasure[2]
|
|
|
|
return sumTotal
|
|
|
|
except Exception as err:
|
|
print("ERROR: Unable to retrieve data for " + metricType + str(err.__dict__))
|
|
sumTotal=0
|
|
return 0
|
|
|
|
def getUsageDataByProject(projectID,startTime,endTime):
|
|
print("Inside getUsageDataByProject {} {} {}".format(projectID,startTime,endTime))
|
|
sumTotal=0
|
|
returnObject = {}
|
|
|
|
returnObject['DOWNLOAD_PEERING']=getUsageDataByProjectandMetricType(projectID,startTime,endTime,"DOWNLOAD_PEERING")
|
|
returnObject['DOWNLOAD_TRANSIT']=getUsageDataByProjectandMetricType(projectID,startTime,endTime,"DOWNLOAD_TRANSIT")
|
|
returnObject['UPLOAD_PEERING']=getUsageDataByProjectandMetricType(projectID,startTime,endTime,"UPLOAD_PEERING")
|
|
returnObject['UPLOAD_TRANSIT']=getUsageDataByProjectandMetricType(projectID,startTime,endTime,"UPLOAD_TRANSIT")
|
|
returnObject['DOWNLOAD_SERVICE']=getUsageDataByProjectandMetricType(projectID,startTime,endTime,"DOWNLOAD_SERVICE")
|
|
returnObject['UPLOAD_SERVICE']=getUsageDataByProjectandMetricType(projectID,startTime,endTime,"UPLOAD_SERVICE")
|
|
|
|
returnObject['SUM_TOTAL']=returnObject['DOWNLOAD_PEERING']+returnObject['DOWNLOAD_TRANSIT']+returnObject['UPLOAD_PEERING']+returnObject['UPLOAD_TRANSIT']
|
|
|
|
print(returnObject)
|
|
return returnObject
|
|
|
|
def createMetric(metricName,resourceID):
|
|
# resourceID=PREFIX+resourceID
|
|
if appDebug:print("Creating metric {} for resourceID {}".format(metricName,resourceID))
|
|
# url='http://'+os.getenv("GNOCCHI_HOST")+':8041/v1/resource/generic/' + resourceID + '/metric'
|
|
data = {metricName: {
|
|
'archive_policy_name': "high"
|
|
}
|
|
}
|
|
|
|
# headers = {'Content-type': 'application/json', 'X-Auth-Token': token}
|
|
# response = requests.post(url, data=data_json, headers=headers)
|
|
response = ng_openstack.openstackRequest.openstackRequest("POST",
|
|
'resource/generic/' + resourceID + '/metric',
|
|
data,
|
|
'http://' + os.getenv("GNOCCHI_HOST") + ':8041/v1')
|
|
|
|
if response.status_code == 200:
|
|
if appDebug:print("Created metric {} for resourceID{}".format(metricName,resourceID))
|
|
for thisMetric in response.json():
|
|
if appDebug:print (thisMetric["name"])
|
|
if thisMetric["name"] == metricName:
|
|
return thisMetric["id"]
|
|
else:
|
|
print ("error in response creating metric " + str(response.status_code))
|
|
print(response.text)
|
|
|
|
def findMetric(metricName, resourceID):
|
|
# resourceID = PREFIX + resourceID
|
|
if appDebug:print("Searching for metric {} in resource {}".format(metricName, resourceID))
|
|
data = {'=': {
|
|
'id': resourceID
|
|
}
|
|
}
|
|
|
|
|
|
# headers = {'Content-type': 'application/json', 'X-Auth-Token': token}
|
|
# url='http://'+os.getenv("GNOCCHI_HOST")+':8041/v1/search/resource/generic'
|
|
# response = requests.post(url, data=data_json, headers=headers)
|
|
response = ng_openstack.openstackRequest.openstackRequest("POST",
|
|
'search/resource/generic',
|
|
data,
|
|
'http://' + os.getenv("GNOCCHI_HOST") + ':8041/v1')
|
|
if response.status_code == 200:
|
|
if len(response.json()) == 1:
|
|
foundresource=response.json()[0]
|
|
resourceID=foundresource["id"]
|
|
|
|
# if 'metrics' not in foundresource:
|
|
# raise ValueError("No metrics in this resource")
|
|
if metricName in foundresource["metrics"]:
|
|
if appDebug: print ("Found existing metric {}".format(foundresource["metrics"][metricName]))
|
|
return foundresource["metrics"][metricName]
|
|
else:
|
|
if appDebug:print ("No metric '%s' found for projectID %s, creating" % (metricName, resourceID))
|
|
return createMetric(metricName,resourceID)
|
|
|
|
else:
|
|
#Resource does not exist, create it
|
|
if appDebug: print ("Resource not found for project ID %s, creating it" % resourceID)
|
|
data = {
|
|
'id': resourceID
|
|
}
|
|
|
|
#data_json = json.dumps(data)
|
|
# headers = {'Content-type': 'application/json', 'X-Auth-Token': token}
|
|
# url='http://'+os.getenv("GNOCCHI_HOST")+':8041/v1/resource/generic'
|
|
# response = requests.post(url, data=data_json, headers=headers)
|
|
response = ng_openstack.openstackRequest.openstackRequest("POST",
|
|
'resource/generic',
|
|
data,
|
|
'http://' + os.getenv("GNOCCHI_HOST") + ':8041/v1')
|
|
|
|
if response.status_code == 201: #201 - Created
|
|
#if appDebug: print ("Resource created ok")
|
|
|
|
if appDebug: print("Created project resource for project ID: %s" % resourceID)
|
|
return findMetric(metricName, resourceID)
|
|
|
|
else:
|
|
raise ValueError ("Error create new resource "+ str(response.status_code) + response.text)
|
|
|
|
|
|
else:
|
|
raise ValueError("error in response finding metric" + str(response.status_code) + response.text)
|
|
|
|
def checkHeliostack_bandwidth_meterResourceExists(thisFlowMetadata, ipResult):
|
|
# Don't prepend the prefix here because it's already been done by the function that calls this function
|
|
# resourceID=PREFIX+resourceID
|
|
resourceID=""
|
|
|
|
#if appDebug: print("Searching for resource for {},{},{} ".format(thisFlowMetadata.billingIP, ipResult.customerID,thisFlowMetadata.routerIP))
|
|
|
|
try:
|
|
resourceID=myAccountingRedisCon.lookupResourceExists(thisFlowMetadata, ipResult)
|
|
#print("Got redis cache hit for resource {}".format(resourceID))
|
|
return resourceID
|
|
|
|
except exceptions.ItemNotFoundError as error:
|
|
pass
|
|
|
|
data = {
|
|
"and": [
|
|
{
|
|
"=": {
|
|
"project_id": ipResult.customerID
|
|
}
|
|
},
|
|
{
|
|
"=": {
|
|
"netFlowSource": thisFlowMetadata.routerIP
|
|
}
|
|
},
|
|
{
|
|
"=": {
|
|
"ipAddress": thisFlowMetadata.billingIP
|
|
}
|
|
}
|
|
]
|
|
}
|
|
#data_json = json.dumps(data)
|
|
|
|
# headers = {'Content-type': 'application/json', 'X-Auth-Token': token}
|
|
# url='http://'+os.getenv("GNOCCHI_HOST")+':8041/v1/search/resource/heliostack_bandwidth_meter'
|
|
# response = requests.post(url, data=data_json, headers=headers)
|
|
response = ng_openstack.openstackRequest.openstackRequest("POST",
|
|
'search/resource/heliostack_bandwidth_meter',
|
|
data,
|
|
'http://' + os.getenv("GNOCCHI_HOST") + ':8041/v1')
|
|
|
|
|
|
if response.status_code == 200:
|
|
if len(response.json()) == 1:
|
|
foundresource=response.json()[0]
|
|
resourceID=foundresource["id"]
|
|
if appDebug:print("Resource found in Gnocchi")
|
|
myAccountingRedisCon.saveResourceExists(thisFlowMetadata, ipResult,resourceID)
|
|
return resourceID
|
|
|
|
else:
|
|
# Resource does not exist, create it
|
|
resourceID = ipResult.customerID + ":" + \
|
|
thisFlowMetadata.billingIP + ":" + thisFlowMetadata.routerIP
|
|
if appDebug: print ("Resource not found for {}, {}, {} ... Creating one".format(
|
|
thisFlowMetadata.billingIP, ipResult.customerID,thisFlowMetadata.routerIP))
|
|
|
|
# The value specified below for 'id' will become 'original_resource_id' and at the time of
|
|
# writing is never used but needs to be filled so we fill with meaningful data
|
|
data = {
|
|
"ipAddress": thisFlowMetadata.billingIP,
|
|
"netFlowSource": thisFlowMetadata.routerIP,
|
|
"id": resourceID,
|
|
"project_id": ipResult.customerID
|
|
}
|
|
|
|
#data_json = json.dumps(data)
|
|
# headers = {'Content-type': 'application/json', 'X-Auth-Token': token}
|
|
# url='http://'+os.getenv("GNOCCHI_HOST")+':8041/v1/resource/heliostack_bandwidth_meter'
|
|
# response = requests.post(url, data=data_json, headers=headers)
|
|
response = ng_openstack.openstackRequest.openstackRequest("POST",
|
|
'resource/heliostack_bandwidth_meter',
|
|
data,
|
|
'http://' + os.getenv("GNOCCHI_HOST") + ':8041/v1')
|
|
|
|
if response.status_code == 201: #201 - Created
|
|
print(response.json())
|
|
foundresource = response.json()
|
|
resourceID = foundresource["id"]
|
|
if appDebug: print ("Resource created ok {}".format(resourceID))
|
|
myAccountingRedisCon.saveResourceExists(thisFlowMetadata, ipResult, resourceID)
|
|
return resourceID
|
|
|
|
else:
|
|
raise ValueError("Error create new resource " + str(response.status_code) + response.text)
|
|
|
|
else:
|
|
raise ValueError("Error in response finding metric" + str(response.status_code) + response.text)
|
|
|
|
def putMeasure(metricID,measureValue,timeStamp):
|
|
if appDebug:print ("Putting measure into metricID:"+metricID)
|
|
data = [{
|
|
"timestamp": timeStamp,
|
|
"value": measureValue
|
|
}]
|
|
|
|
#data_json = json.dumps(data)
|
|
print(json.dumps(data))
|
|
# headers = {'Content-type': 'application/json', 'X-Auth-Token': token}
|
|
# url='http://'+os.getenv("GNOCCHI_HOST")+':8041/v1/metric/' + metricID + '/measures'
|
|
# response = requests.post(url, data=data_json, headers=headers)
|
|
response = ng_openstack.openstackRequest.openstackRequest("POST",
|
|
'metric/' + metricID + '/measures',
|
|
data, 'http://' + os.getenv("GNOCCHI_HOST") + ':8041/v1')
|
|
|
|
if response.status_code == 202:
|
|
if appDebug: print("Added data to metric metricID:" + metricID)
|
|
else:
|
|
print(response.__dict__)
|
|
raise ValueError ("error in response while putting data" + str(response.status_code))
|
|
|
|
def putBulkMeasure(bulkData):
|
|
if appDebug:print ("Putting bulk measure data")
|
|
if len(bulkData)<3:
|
|
print("********SHORT BULK MEASURE DATA********")
|
|
|
|
#data_json = json.dumps(bulkData)
|
|
print(json.dumps(bulkData))
|
|
# headers = {'Content-type': 'application/json', 'X-Auth-Token': token}
|
|
# url='http://'+os.getenv("GNOCCHI_HOST")+':8041/v1/batch/resources/metrics/measures?create_metrics=true'
|
|
|
|
response=ng_openstack.openstackRequest.openstackRequest("POST","batch/resources/metrics/measures?create_metrics=true",bulkData,'http://'+os.getenv("GNOCCHI_HOST")+':8041/v1')
|
|
# response = requests.post(url, data=data_json, headers=headers)
|
|
|
|
if response.status_code == 202:
|
|
print ("Added bulk data")
|
|
else:
|
|
print(response.__dict__)
|
|
raise ValueError ("Error in response while putting bulk data" + str(response.status_code))
|
|
|
|
def getHeliostack_measureResourceID(projectID):
|
|
|
|
data = {
|
|
{
|
|
"=": {
|
|
"project_id": projectID
|
|
}
|
|
}
|
|
}
|
|
|
|
response = ng_openstack.openstackRequest.openstackRequest("POST",
|
|
'search/resource/heliostack_measure',
|
|
data,
|
|
'http://' + os.getenv("GNOCCHI_HOST") + ':8041/v1')
|
|
|
|
|
|
if response.status_code == 200:
|
|
if len(response.json()) == 1:
|
|
foundresource=response.json()[0]
|
|
resourceID=foundresource["id"]
|
|
if appDebug:print("Resource found in Gnocchi")
|
|
myAccountingRedisCon.saveResourceExists(thisFlowMetadata, ipResult,resourceID)
|
|
return resourceID
|
|
|
|
else:
|
|
# Resource does not exist, create it
|
|
resourceID = ipResult.customerID + ":" + \
|
|
thisFlowMetadata.billingIP + ":" + thisFlowMetadata.routerIP
|
|
if appDebug: print ("Resource not found for {}, {}, {} ... Creating one".format(
|
|
thisFlowMetadata.billingIP, ipResult.customerID,thisFlowMetadata.routerIP))
|
|
|
|
# The value specified below for 'id' will become 'original_resource_id' and at the time of
|
|
# writing is never used but needs to be filled so we fill with meaningful data
|
|
data = {
|
|
"ipAddress": thisFlowMetadata.billingIP,
|
|
"netFlowSource": thisFlowMetadata.routerIP,
|
|
"id": resourceID,
|
|
"project_id": ipResult.customerID
|
|
}
|
|
|
|
#data_json = json.dumps(data)
|
|
# headers = {'Content-type': 'application/json', 'X-Auth-Token': token}
|
|
# url='http://'+os.getenv("GNOCCHI_HOST")+':8041/v1/resource/heliostack_bandwidth_meter'
|
|
# response = requests.post(url, data=data_json, headers=headers)
|
|
response = ng_openstack.openstackRequest.openstackRequest("POST",
|
|
'resource/heliostack_bandwidth_meter',
|
|
data,
|
|
'http://' + os.getenv("GNOCCHI_HOST") + ':8041/v1')
|
|
|
|
if response.status_code == 201: #201 - Created
|
|
print(response.json())
|
|
foundresource = response.json()
|
|
resourceID = foundresource["id"]
|
|
if appDebug: print ("Resource created ok {}".format(resourceID))
|
|
myAccountingRedisCon.saveResourceExists(thisFlowMetadata, ipResult, resourceID)
|
|
return resourceID
|
|
|
|
else:
|
|
raise ValueError("Error create new resource " + str(response.status_code) + response.text)
|
|
|
|
else:
|
|
raise ValueError("Error in response finding metric" + str(response.status_code) + response.text)
|
|
|
|
def checkheliostack_consumable_resourceExists(projectID,projectName):
|
|
data = {
|
|
"=": {
|
|
"project_id": projectID
|
|
}
|
|
}
|
|
|
|
|
|
|
|
response = ng_openstack.openstackRequest.openstackRequest("POST",
|
|
'search/resource/heliostack_consumable_resource',
|
|
data,
|
|
'http://' + os.getenv("GNOCCHI_HOST") + ':8041/v1')
|
|
|
|
if response.status_code == 200:
|
|
if len(response.json()) == 1:
|
|
foundresource=response.json()[0]
|
|
resourceID=foundresource["id"]
|
|
if appDebug:print("Resource found in Gnocchi")
|
|
return resourceID
|
|
|
|
else:
|
|
# Resource does not exist, create it
|
|
|
|
if appDebug: print ("Resource not found for project {} ... Creating one".format(projectID))
|
|
|
|
# The value specified below for 'id' will become 'original_resource_id' and at the time of
|
|
# writing is never used but needs to be filled so we fill with meaningful data
|
|
data = {
|
|
"id": projectID,
|
|
"project_id": projectID,
|
|
"project_name": projectName
|
|
}
|
|
|
|
response = ng_openstack.openstackRequest.openstackRequest("POST",
|
|
'resource/heliostack_consumable_resource',
|
|
data,
|
|
'http://' + os.getenv("GNOCCHI_HOST") + ':8041/v1')
|
|
|
|
if response.status_code == 201: #201 - Created
|
|
print(response.json())
|
|
foundresource = response.json()
|
|
resourceID = foundresource["id"]
|
|
if appDebug: print ("Resource created ok {}".format(resourceID))
|
|
return resourceID
|
|
|
|
else:
|
|
raise ValueError("Error create new resource " + str(response.status_code) + response.text)
|
|
|
|
else:
|
|
raise ValueError("Error in response finding metric" + str(response.status_code) + response.text) |