import settings,os import ng_openstack.openstackRequest import exceptions import json import pybill_redis import requests import datetime, pytz, calendar appDebug = True myAccountingRedisCon=pybill_redis.pyRedis(os.getenv("REDIS_HOST"),6379,"") PREFIX="pyBill_" def add_months(date, months): months_count = date.month + months # Calculate the year year = date.year + int(months_count / 12) # Calculate the month month = (months_count % 12) if month == 0: month = 12 # Calculate the day day = date.day last_day_of_month = calendar.monthrange(year, month)[1] if day > last_day_of_month: day = last_day_of_month new_date = datetime.date(year, month, day) return new_date def getBWUsageByMonthforProject(yearNumber, monthNumber, projectID): fmt = '%Y-%m-%dT%H:%M:%S' # YY,MM,DD reportingMonth = datetime.datetime(int(yearNumber), int(monthNumber), 1) midnightOnFirstDayofMonthInUTC = datetime.datetime.now(pytz.timezone("UTC")) \ .replace(year=reportingMonth.year, month=reportingMonth.month, day=1, hour=0, minute=0, second=0, microsecond=0) \ .astimezone(pytz.utc) startTime = midnightOnFirstDayofMonthInUTC.strftime(fmt) # Should we roll to next year? theNextMonth = add_months(reportingMonth, 1).month theNextYear = add_months(reportingMonth, 1).year midnightOnFirstDayofFollowingMonthInUTC = datetime.datetime.now(pytz.timezone("UTC")) \ .replace(year=theNextYear, month=theNextMonth, day=1, hour=0, minute=0, second=0, microsecond=0) \ .astimezone(pytz.utc) endTime = midnightOnFirstDayofFollowingMonthInUTC.strftime(fmt) return getUsageDataByProject(projectID,startTime,endTime) def getUsageDataByProjectandMetricType(projectID,startTime,endTime,metricType): data = { "operations": [ "aggregate", "sum", [ "metric", metricType, "sum" ] ], "resource_type": "heliostack_bandwidth_meter", "search": { "=": { "project_id":projectID } } } try: usageData = ng_openstack.openstackRequest.openstackRequest("POST", "aggregates?granularity=3600&start="+startTime+"&stop="+endTime,data, 'http://' + os.getenv("GNOCCHI_HOST") + ':8041/v1') # print(usageData.json()) # print(str(len(usageData.json()['measures']['aggregated'])) + " Entries") sumTotal = 0 for thisMeasure in usageData.json()['measures']['aggregated']: if thisMeasure[2]>1000000000000: thisMeasure[2]=0 if thisMeasure[2]<0: thisMeasure[2]=0 sumTotal += thisMeasure[2] return sumTotal except Exception as err: print("ERROR: Unable to retrieve data for " + metricType + str(err.__dict__)) sumTotal=0 return 0 def getUsageDataByProject(projectID,startTime,endTime): print("Inside getUsageDataByProject {} {} {}".format(projectID,startTime,endTime)) sumTotal=0 returnObject = {} returnObject['DOWNLOAD_PEERING']=getUsageDataByProjectandMetricType(projectID,startTime,endTime,"DOWNLOAD_PEERING") returnObject['DOWNLOAD_TRANSIT']=getUsageDataByProjectandMetricType(projectID,startTime,endTime,"DOWNLOAD_TRANSIT") returnObject['UPLOAD_PEERING']=getUsageDataByProjectandMetricType(projectID,startTime,endTime,"UPLOAD_PEERING") returnObject['UPLOAD_TRANSIT']=getUsageDataByProjectandMetricType(projectID,startTime,endTime,"UPLOAD_TRANSIT") returnObject['DOWNLOAD_SERVICE']=getUsageDataByProjectandMetricType(projectID,startTime,endTime,"DOWNLOAD_SERVICE") returnObject['UPLOAD_SERVICE']=getUsageDataByProjectandMetricType(projectID,startTime,endTime,"UPLOAD_SERVICE") returnObject['SUM_TOTAL']=returnObject['DOWNLOAD_PEERING']+returnObject['DOWNLOAD_TRANSIT']+returnObject['UPLOAD_PEERING']+returnObject['UPLOAD_TRANSIT'] print(returnObject) return returnObject def createMetric(metricName,resourceID): # resourceID=PREFIX+resourceID if appDebug:print("Creating metric {} for resourceID {}".format(metricName,resourceID)) # url='http://'+os.getenv("GNOCCHI_HOST")+':8041/v1/resource/generic/' + resourceID + '/metric' data = {metricName: { 'archive_policy_name': "high" } } # headers = {'Content-type': 'application/json', 'X-Auth-Token': token} # response = requests.post(url, data=data_json, headers=headers) response = ng_openstack.openstackRequest.openstackRequest("POST", 'resource/generic/' + resourceID + '/metric', data, 'http://' + os.getenv("GNOCCHI_HOST") + ':8041/v1') if response.status_code == 200: if appDebug:print("Created metric {} for resourceID{}".format(metricName,resourceID)) for thisMetric in response.json(): if appDebug:print (thisMetric["name"]) if thisMetric["name"] == metricName: return thisMetric["id"] else: print ("error in response creating metric " + str(response.status_code)) print(response.text) def findMetric(metricName, resourceID): # resourceID = PREFIX + resourceID if appDebug:print("Searching for metric {} in resource {}".format(metricName, resourceID)) data = {'=': { 'id': resourceID } } # headers = {'Content-type': 'application/json', 'X-Auth-Token': token} # url='http://'+os.getenv("GNOCCHI_HOST")+':8041/v1/search/resource/generic' # response = requests.post(url, data=data_json, headers=headers) response = ng_openstack.openstackRequest.openstackRequest("POST", 'search/resource/generic', data, 'http://' + os.getenv("GNOCCHI_HOST") + ':8041/v1') if response.status_code == 200: if len(response.json()) == 1: foundresource=response.json()[0] resourceID=foundresource["id"] # if 'metrics' not in foundresource: # raise ValueError("No metrics in this resource") if metricName in foundresource["metrics"]: if appDebug: print ("Found existing metric {}".format(foundresource["metrics"][metricName])) return foundresource["metrics"][metricName] else: if appDebug:print ("No metric '%s' found for projectID %s, creating" % (metricName, resourceID)) return createMetric(metricName,resourceID) else: #Resource does not exist, create it if appDebug: print ("Resource not found for project ID %s, creating it" % resourceID) data = { 'id': resourceID } #data_json = json.dumps(data) # headers = {'Content-type': 'application/json', 'X-Auth-Token': token} # url='http://'+os.getenv("GNOCCHI_HOST")+':8041/v1/resource/generic' # response = requests.post(url, data=data_json, headers=headers) response = ng_openstack.openstackRequest.openstackRequest("POST", 'resource/generic', data, 'http://' + os.getenv("GNOCCHI_HOST") + ':8041/v1') if response.status_code == 201: #201 - Created #if appDebug: print ("Resource created ok") if appDebug: print("Created project resource for project ID: %s" % resourceID) return findMetric(metricName, resourceID) else: raise ValueError ("Error create new resource "+ str(response.status_code) + response.text) else: raise ValueError("error in response finding metric" + str(response.status_code) + response.text) def checkHeliostack_bandwidth_meterResourceExists(thisFlowMetadata, ipResult): # Don't prepend the prefix here because it's already been done by the function that calls this function # resourceID=PREFIX+resourceID resourceID="" #if appDebug: print("Searching for resource for {},{},{} ".format(thisFlowMetadata.billingIP, ipResult.customerID,thisFlowMetadata.routerIP)) try: resourceID=myAccountingRedisCon.lookupResourceExists(thisFlowMetadata, ipResult) #print("Got redis cache hit for resource {}".format(resourceID)) return resourceID except exceptions.ItemNotFoundError as error: pass data = { "and": [ { "=": { "project_id": ipResult.customerID } }, { "=": { "netFlowSource": thisFlowMetadata.routerIP } }, { "=": { "ipAddress": thisFlowMetadata.billingIP } } ] } #data_json = json.dumps(data) # headers = {'Content-type': 'application/json', 'X-Auth-Token': token} # url='http://'+os.getenv("GNOCCHI_HOST")+':8041/v1/search/resource/heliostack_bandwidth_meter' # response = requests.post(url, data=data_json, headers=headers) response = ng_openstack.openstackRequest.openstackRequest("POST", 'search/resource/heliostack_bandwidth_meter', data, 'http://' + os.getenv("GNOCCHI_HOST") + ':8041/v1') if response.status_code == 200: if len(response.json()) == 1: foundresource=response.json()[0] resourceID=foundresource["id"] if appDebug:print("Resource found in Gnocchi") myAccountingRedisCon.saveResourceExists(thisFlowMetadata, ipResult,resourceID) return resourceID else: # Resource does not exist, create it resourceID = ipResult.customerID + ":" + \ thisFlowMetadata.billingIP + ":" + thisFlowMetadata.routerIP if appDebug: print ("Resource not found for {}, {}, {} ... Creating one".format( thisFlowMetadata.billingIP, ipResult.customerID,thisFlowMetadata.routerIP)) # The value specified below for 'id' will become 'original_resource_id' and at the time of # writing is never used but needs to be filled so we fill with meaningful data data = { "ipAddress": thisFlowMetadata.billingIP, "netFlowSource": thisFlowMetadata.routerIP, "id": resourceID, "project_id": ipResult.customerID } #data_json = json.dumps(data) # headers = {'Content-type': 'application/json', 'X-Auth-Token': token} # url='http://'+os.getenv("GNOCCHI_HOST")+':8041/v1/resource/heliostack_bandwidth_meter' # response = requests.post(url, data=data_json, headers=headers) response = ng_openstack.openstackRequest.openstackRequest("POST", 'resource/heliostack_bandwidth_meter', data, 'http://' + os.getenv("GNOCCHI_HOST") + ':8041/v1') if response.status_code == 201: #201 - Created print(response.json()) foundresource = response.json() resourceID = foundresource["id"] if appDebug: print ("Resource created ok {}".format(resourceID)) myAccountingRedisCon.saveResourceExists(thisFlowMetadata, ipResult, resourceID) return resourceID else: raise ValueError("Error create new resource " + str(response.status_code) + response.text) else: raise ValueError("Error in response finding metric" + str(response.status_code) + response.text) def putMeasure(metricID,measureValue,timeStamp): if appDebug:print ("Putting measure into metricID:"+metricID) data = [{ "timestamp": timeStamp, "value": measureValue }] #data_json = json.dumps(data) print(json.dumps(data)) # headers = {'Content-type': 'application/json', 'X-Auth-Token': token} # url='http://'+os.getenv("GNOCCHI_HOST")+':8041/v1/metric/' + metricID + '/measures' # response = requests.post(url, data=data_json, headers=headers) response = ng_openstack.openstackRequest.openstackRequest("POST", 'metric/' + metricID + '/measures', data, 'http://' + os.getenv("GNOCCHI_HOST") + ':8041/v1') if response.status_code == 202: if appDebug: print("Added data to metric metricID:" + metricID) else: print(response.__dict__) raise ValueError ("error in response while putting data" + str(response.status_code)) def putBulkMeasure(bulkData): if appDebug:print ("Putting bulk measure data") if len(bulkData)<3: print("********SHORT BULK MEASURE DATA********") #data_json = json.dumps(bulkData) print(json.dumps(bulkData)) # headers = {'Content-type': 'application/json', 'X-Auth-Token': token} # url='http://'+os.getenv("GNOCCHI_HOST")+':8041/v1/batch/resources/metrics/measures?create_metrics=true' response=ng_openstack.openstackRequest.openstackRequest("POST","batch/resources/metrics/measures?create_metrics=true",bulkData,'http://'+os.getenv("GNOCCHI_HOST")+':8041/v1') # response = requests.post(url, data=data_json, headers=headers) if response.status_code == 202: print ("Added bulk data") else: print(response.__dict__) raise ValueError ("Error in response while putting bulk data" + str(response.status_code)) def getHeliostack_measureResourceID(projectID): data = { { "=": { "project_id": projectID } } } response = ng_openstack.openstackRequest.openstackRequest("POST", 'search/resource/heliostack_measure', data, 'http://' + os.getenv("GNOCCHI_HOST") + ':8041/v1') if response.status_code == 200: if len(response.json()) == 1: foundresource=response.json()[0] resourceID=foundresource["id"] if appDebug:print("Resource found in Gnocchi") myAccountingRedisCon.saveResourceExists(thisFlowMetadata, ipResult,resourceID) return resourceID else: # Resource does not exist, create it resourceID = ipResult.customerID + ":" + \ thisFlowMetadata.billingIP + ":" + thisFlowMetadata.routerIP if appDebug: print ("Resource not found for {}, {}, {} ... Creating one".format( thisFlowMetadata.billingIP, ipResult.customerID,thisFlowMetadata.routerIP)) # The value specified below for 'id' will become 'original_resource_id' and at the time of # writing is never used but needs to be filled so we fill with meaningful data data = { "ipAddress": thisFlowMetadata.billingIP, "netFlowSource": thisFlowMetadata.routerIP, "id": resourceID, "project_id": ipResult.customerID } #data_json = json.dumps(data) # headers = {'Content-type': 'application/json', 'X-Auth-Token': token} # url='http://'+os.getenv("GNOCCHI_HOST")+':8041/v1/resource/heliostack_bandwidth_meter' # response = requests.post(url, data=data_json, headers=headers) response = ng_openstack.openstackRequest.openstackRequest("POST", 'resource/heliostack_bandwidth_meter', data, 'http://' + os.getenv("GNOCCHI_HOST") + ':8041/v1') if response.status_code == 201: #201 - Created print(response.json()) foundresource = response.json() resourceID = foundresource["id"] if appDebug: print ("Resource created ok {}".format(resourceID)) myAccountingRedisCon.saveResourceExists(thisFlowMetadata, ipResult, resourceID) return resourceID else: raise ValueError("Error create new resource " + str(response.status_code) + response.text) else: raise ValueError("Error in response finding metric" + str(response.status_code) + response.text) def checkheliostack_consumable_resourceExists(projectID,projectName): data = { "=": { "project_id": projectID } } response = ng_openstack.openstackRequest.openstackRequest("POST", 'search/resource/heliostack_consumable_resource', data, 'http://' + os.getenv("GNOCCHI_HOST") + ':8041/v1') if response.status_code == 200: if len(response.json()) == 1: foundresource=response.json()[0] resourceID=foundresource["id"] if appDebug:print("Resource found in Gnocchi") return resourceID else: # Resource does not exist, create it if appDebug: print ("Resource not found for project {} ... Creating one".format(projectID)) # The value specified below for 'id' will become 'original_resource_id' and at the time of # writing is never used but needs to be filled so we fill with meaningful data data = { "id": projectID, "project_id": projectID, "project_name": projectName } response = ng_openstack.openstackRequest.openstackRequest("POST", 'resource/heliostack_consumable_resource', data, 'http://' + os.getenv("GNOCCHI_HOST") + ':8041/v1') if response.status_code == 201: #201 - Created print(response.json()) foundresource = response.json() resourceID = foundresource["id"] if appDebug: print ("Resource created ok {}".format(resourceID)) return resourceID else: raise ValueError("Error create new resource " + str(response.status_code) + response.text) else: raise ValueError("Error in response finding metric" + str(response.status_code) + response.text)