new-openstackcheck/ng_openstack/gnocchi.py

452 lines
19 KiB
Python

import settings,os
import ng_openstack.openstackRequest
import ng_openstack.exceptions as exceptions
import json
import pybill_redis
import requests
import datetime, pytz, calendar
appDebug = True
myAccountingRedisCon=pybill_redis.pyRedis(os.getenv("REDIS_HOST"),6379,"")
PREFIX="pyBill_"
def add_months(date, months):
months_count = date.month + months
# Calculate the year
year = date.year + int(months_count / 12)
# Calculate the month
month = (months_count % 12)
if month == 0:
month = 12
# Calculate the day
day = date.day
last_day_of_month = calendar.monthrange(year, month)[1]
if day > last_day_of_month:
day = last_day_of_month
new_date = datetime.date(year, month, day)
return new_date
def getBWUsageByMonthforProject(yearNumber, monthNumber, projectID):
fmt = '%Y-%m-%dT%H:%M:%S'
# YY,MM,DD
reportingMonth = datetime.datetime(int(yearNumber), int(monthNumber), 1)
midnightOnFirstDayofMonthInUTC = datetime.datetime.now(pytz.timezone("UTC")) \
.replace(year=reportingMonth.year, month=reportingMonth.month, day=1, hour=0, minute=0, second=0, microsecond=0) \
.astimezone(pytz.utc)
startTime = midnightOnFirstDayofMonthInUTC.strftime(fmt)
# Should we roll to next year?
theNextMonth = add_months(reportingMonth, 1).month
theNextYear = add_months(reportingMonth, 1).year
midnightOnFirstDayofFollowingMonthInUTC = datetime.datetime.now(pytz.timezone("UTC")) \
.replace(year=theNextYear, month=theNextMonth, day=1, hour=0, minute=0, second=0,
microsecond=0) \
.astimezone(pytz.utc)
endTime = midnightOnFirstDayofFollowingMonthInUTC.strftime(fmt)
return getUsageDataByProject(projectID,startTime,endTime)
def getUsageDataByProjectandMetricType(projectID,startTime,endTime,metricType):
data = {
"operations": [
"aggregate",
"sum",
[
"metric",
metricType,
"sum"
]
],
"resource_type": "heliostack_bandwidth_meter",
"search": {
"=": {
"project_id":projectID
}
}
}
try:
usageData = ng_openstack.openstackRequest.openstackRequest("POST", "aggregates?granularity=3600&start="+startTime+"&stop="+endTime,data,
'http://' + os.getenv("GNOCCHI_HOST") + ':8041/v1')
# print(usageData.json())
# print(str(len(usageData.json()['measures']['aggregated'])) + " Entries")
sumTotal = 0
for thisMeasure in usageData.json()['measures']['aggregated']:
if thisMeasure[2]>1000000000000:
thisMeasure[2]=0
if thisMeasure[2]<0:
thisMeasure[2]=0
sumTotal += thisMeasure[2]
return sumTotal
except Exception as err:
print("ERROR: Unable to retrieve data for " + metricType + str(err.__dict__))
sumTotal=0
return 0
def getUsageDataByProject(projectID,startTime,endTime):
print("Inside getUsageDataByProject {} {} {}".format(projectID,startTime,endTime))
sumTotal=0
returnObject = {}
returnObject['DOWNLOAD_PEERING']=getUsageDataByProjectandMetricType(projectID,startTime,endTime,"DOWNLOAD_PEERING")
returnObject['DOWNLOAD_TRANSIT']=getUsageDataByProjectandMetricType(projectID,startTime,endTime,"DOWNLOAD_TRANSIT")
returnObject['UPLOAD_PEERING']=getUsageDataByProjectandMetricType(projectID,startTime,endTime,"UPLOAD_PEERING")
returnObject['UPLOAD_TRANSIT']=getUsageDataByProjectandMetricType(projectID,startTime,endTime,"UPLOAD_TRANSIT")
returnObject['DOWNLOAD_SERVICE']=getUsageDataByProjectandMetricType(projectID,startTime,endTime,"DOWNLOAD_SERVICE")
returnObject['UPLOAD_SERVICE']=getUsageDataByProjectandMetricType(projectID,startTime,endTime,"UPLOAD_SERVICE")
returnObject['SUM_TOTAL']=returnObject['DOWNLOAD_PEERING']+returnObject['DOWNLOAD_TRANSIT']+returnObject['UPLOAD_PEERING']+returnObject['UPLOAD_TRANSIT']
print(returnObject)
return returnObject
def createMetric(metricName,resourceID):
# resourceID=PREFIX+resourceID
if appDebug:print("Creating metric {} for resourceID {}".format(metricName,resourceID))
# url='http://'+os.getenv("GNOCCHI_HOST")+':8041/v1/resource/generic/' + resourceID + '/metric'
data = {metricName: {
'archive_policy_name': "high"
}
}
# headers = {'Content-type': 'application/json', 'X-Auth-Token': token}
# response = requests.post(url, data=data_json, headers=headers)
response = ng_openstack.openstackRequest.openstackRequest("POST",
'resource/generic/' + resourceID + '/metric',
data,
'http://' + os.getenv("GNOCCHI_HOST") + ':8041/v1')
if response.status_code == 200:
if appDebug:print("Created metric {} for resourceID{}".format(metricName,resourceID))
for thisMetric in response.json():
if appDebug:print (thisMetric["name"])
if thisMetric["name"] == metricName:
return thisMetric["id"]
else:
print ("error in response creating metric " + str(response.status_code))
print(response.text)
def findMetric(metricName, resourceID):
# resourceID = PREFIX + resourceID
if appDebug:print("Searching for metric {} in resource {}".format(metricName, resourceID))
data = {'=': {
'id': resourceID
}
}
# headers = {'Content-type': 'application/json', 'X-Auth-Token': token}
# url='http://'+os.getenv("GNOCCHI_HOST")+':8041/v1/search/resource/generic'
# response = requests.post(url, data=data_json, headers=headers)
response = ng_openstack.openstackRequest.openstackRequest("POST",
'search/resource/generic',
data,
'http://' + os.getenv("GNOCCHI_HOST") + ':8041/v1')
if response.status_code == 200:
if len(response.json()) == 1:
foundresource=response.json()[0]
resourceID=foundresource["id"]
# if 'metrics' not in foundresource:
# raise ValueError("No metrics in this resource")
if metricName in foundresource["metrics"]:
if appDebug: print ("Found existing metric {}".format(foundresource["metrics"][metricName]))
return foundresource["metrics"][metricName]
else:
if appDebug:print ("No metric '%s' found for projectID %s, creating" % (metricName, resourceID))
return createMetric(metricName,resourceID)
else:
#Resource does not exist, create it
if appDebug: print ("Resource not found for project ID %s, creating it" % resourceID)
data = {
'id': resourceID
}
#data_json = json.dumps(data)
# headers = {'Content-type': 'application/json', 'X-Auth-Token': token}
# url='http://'+os.getenv("GNOCCHI_HOST")+':8041/v1/resource/generic'
# response = requests.post(url, data=data_json, headers=headers)
response = ng_openstack.openstackRequest.openstackRequest("POST",
'resource/generic',
data,
'http://' + os.getenv("GNOCCHI_HOST") + ':8041/v1')
if response.status_code == 201: #201 - Created
#if appDebug: print ("Resource created ok")
if appDebug: print("Created project resource for project ID: %s" % resourceID)
return findMetric(metricName, resourceID)
else:
raise ValueError ("Error create new resource "+ str(response.status_code) + response.text)
else:
raise ValueError("error in response finding metric" + str(response.status_code) + response.text)
def checkHeliostack_bandwidth_meterResourceExists(thisFlowMetadata, ipResult):
# Don't prepend the prefix here because it's already been done by the function that calls this function
# resourceID=PREFIX+resourceID
resourceID=""
#if appDebug: print("Searching for resource for {},{},{} ".format(thisFlowMetadata.billingIP, ipResult.customerID,thisFlowMetadata.routerIP))
try:
resourceID=myAccountingRedisCon.lookupResourceExists(thisFlowMetadata, ipResult)
#print("Got redis cache hit for resource {}".format(resourceID))
return resourceID
except exceptions.ItemNotFoundError as error:
pass
data = {
"and": [
{
"=": {
"project_id": ipResult.customerID
}
},
{
"=": {
"netFlowSource": thisFlowMetadata.routerIP
}
},
{
"=": {
"ipAddress": thisFlowMetadata.billingIP
}
}
]
}
#data_json = json.dumps(data)
# headers = {'Content-type': 'application/json', 'X-Auth-Token': token}
# url='http://'+os.getenv("GNOCCHI_HOST")+':8041/v1/search/resource/heliostack_bandwidth_meter'
# response = requests.post(url, data=data_json, headers=headers)
response = ng_openstack.openstackRequest.openstackRequest("POST",
'search/resource/heliostack_bandwidth_meter',
data,
'http://' + os.getenv("GNOCCHI_HOST") + ':8041/v1')
if response.status_code == 200:
if len(response.json()) == 1:
foundresource=response.json()[0]
resourceID=foundresource["id"]
if appDebug:print("Resource found in Gnocchi")
myAccountingRedisCon.saveResourceExists(thisFlowMetadata, ipResult,resourceID)
return resourceID
else:
# Resource does not exist, create it
resourceID = ipResult.customerID + ":" + \
thisFlowMetadata.billingIP + ":" + thisFlowMetadata.routerIP
if appDebug: print ("Resource not found for {}, {}, {} ... Creating one".format(
thisFlowMetadata.billingIP, ipResult.customerID,thisFlowMetadata.routerIP))
# The value specified below for 'id' will become 'original_resource_id' and at the time of
# writing is never used but needs to be filled so we fill with meaningful data
data = {
"ipAddress": thisFlowMetadata.billingIP,
"netFlowSource": thisFlowMetadata.routerIP,
"id": resourceID,
"project_id": ipResult.customerID
}
#data_json = json.dumps(data)
# headers = {'Content-type': 'application/json', 'X-Auth-Token': token}
# url='http://'+os.getenv("GNOCCHI_HOST")+':8041/v1/resource/heliostack_bandwidth_meter'
# response = requests.post(url, data=data_json, headers=headers)
response = ng_openstack.openstackRequest.openstackRequest("POST",
'resource/heliostack_bandwidth_meter',
data,
'http://' + os.getenv("GNOCCHI_HOST") + ':8041/v1')
if response.status_code == 201: #201 - Created
print(response.json())
foundresource = response.json()
resourceID = foundresource["id"]
if appDebug: print ("Resource created ok {}".format(resourceID))
myAccountingRedisCon.saveResourceExists(thisFlowMetadata, ipResult, resourceID)
return resourceID
else:
raise ValueError("Error create new resource " + str(response.status_code) + response.text)
else:
raise ValueError("Error in response finding metric" + str(response.status_code) + response.text)
def putMeasure(metricID,measureValue,timeStamp):
if appDebug:print ("Putting measure into metricID:"+metricID)
data = [{
"timestamp": timeStamp,
"value": measureValue
}]
#data_json = json.dumps(data)
print(json.dumps(data))
# headers = {'Content-type': 'application/json', 'X-Auth-Token': token}
# url='http://'+os.getenv("GNOCCHI_HOST")+':8041/v1/metric/' + metricID + '/measures'
# response = requests.post(url, data=data_json, headers=headers)
response = ng_openstack.openstackRequest.openstackRequest("POST",
'metric/' + metricID + '/measures',
data, 'http://' + os.getenv("GNOCCHI_HOST") + ':8041/v1')
if response.status_code == 202:
if appDebug: print("Added data to metric metricID:" + metricID)
else:
print(response.__dict__)
raise ValueError ("error in response while putting data" + str(response.status_code))
def putBulkMeasure(bulkData):
if appDebug:print ("Putting bulk measure data")
if len(bulkData)<3:
print("********SHORT BULK MEASURE DATA********")
#data_json = json.dumps(bulkData)
print(json.dumps(bulkData))
# headers = {'Content-type': 'application/json', 'X-Auth-Token': token}
# url='http://'+os.getenv("GNOCCHI_HOST")+':8041/v1/batch/resources/metrics/measures?create_metrics=true'
response=ng_openstack.openstackRequest.openstackRequest("POST","batch/resources/metrics/measures?create_metrics=true",bulkData,'http://'+os.getenv("GNOCCHI_HOST")+':8041/v1')
# response = requests.post(url, data=data_json, headers=headers)
if response.status_code == 202:
print ("Added bulk data")
else:
print(response.__dict__)
raise ValueError ("Error in response while putting bulk data" + str(response.status_code))
def getHeliostack_measureResourceID(projectID):
data = {
{
"=": {
"project_id": projectID
}
}
}
response = ng_openstack.openstackRequest.openstackRequest("POST",
'search/resource/heliostack_measure',
data,
'http://' + os.getenv("GNOCCHI_HOST") + ':8041/v1')
if response.status_code == 200:
if len(response.json()) == 1:
foundresource=response.json()[0]
resourceID=foundresource["id"]
if appDebug:print("Resource found in Gnocchi")
myAccountingRedisCon.saveResourceExists(thisFlowMetadata, ipResult,resourceID)
return resourceID
else:
# Resource does not exist, create it
resourceID = ipResult.customerID + ":" + \
thisFlowMetadata.billingIP + ":" + thisFlowMetadata.routerIP
if appDebug: print ("Resource not found for {}, {}, {} ... Creating one".format(
thisFlowMetadata.billingIP, ipResult.customerID,thisFlowMetadata.routerIP))
# The value specified below for 'id' will become 'original_resource_id' and at the time of
# writing is never used but needs to be filled so we fill with meaningful data
data = {
"ipAddress": thisFlowMetadata.billingIP,
"netFlowSource": thisFlowMetadata.routerIP,
"id": resourceID,
"project_id": ipResult.customerID
}
#data_json = json.dumps(data)
# headers = {'Content-type': 'application/json', 'X-Auth-Token': token}
# url='http://'+os.getenv("GNOCCHI_HOST")+':8041/v1/resource/heliostack_bandwidth_meter'
# response = requests.post(url, data=data_json, headers=headers)
response = ng_openstack.openstackRequest.openstackRequest("POST",
'resource/heliostack_bandwidth_meter',
data,
'http://' + os.getenv("GNOCCHI_HOST") + ':8041/v1')
if response.status_code == 201: #201 - Created
print(response.json())
foundresource = response.json()
resourceID = foundresource["id"]
if appDebug: print ("Resource created ok {}".format(resourceID))
myAccountingRedisCon.saveResourceExists(thisFlowMetadata, ipResult, resourceID)
return resourceID
else:
raise ValueError("Error create new resource " + str(response.status_code) + response.text)
else:
raise ValueError("Error in response finding metric" + str(response.status_code) + response.text)
def checkheliostack_consumable_resourceExists(projectID,projectName):
data = {
"=": {
"project_id": projectID
}
}
response = ng_openstack.openstackRequest.openstackRequest("POST",
'search/resource/heliostack_consumable_resource',
data,
'http://' + os.getenv("GNOCCHI_HOST") + ':8041/v1')
if response.status_code == 200:
if len(response.json()) == 1:
foundresource=response.json()[0]
resourceID=foundresource["id"]
if appDebug:print("Resource found in Gnocchi")
return resourceID
else:
# Resource does not exist, create it
if appDebug: print ("Resource not found for project {} ... Creating one".format(projectID))
# The value specified below for 'id' will become 'original_resource_id' and at the time of
# writing is never used but needs to be filled so we fill with meaningful data
data = {
"id": projectID,
"project_id": projectID,
"project_name": projectName
}
response = ng_openstack.openstackRequest.openstackRequest("POST",
'resource/heliostack_consumable_resource',
data,
'http://' + os.getenv("GNOCCHI_HOST") + ':8041/v1')
if response.status_code == 201: #201 - Created
print(response.json())
foundresource = response.json()
resourceID = foundresource["id"]
if appDebug: print ("Resource created ok {}".format(resourceID))
return resourceID
else:
raise ValueError("Error create new resource " + str(response.status_code) + response.text)
else:
raise ValueError("Error in response finding metric" + str(response.status_code) + response.text)