From 60fe08f7546b72be3aa4ead92cfb770084dd4198 Mon Sep 17 00:00:00 2001 From: Cory Hawkless Date: Mon, 10 Oct 2022 15:55:49 +1030 Subject: [PATCH] framework --- .env.example | 8 + .gitignore | 12 + README.md | 6 + exceptions.py | 7 + main.py | 7 + ng_openstack/__init__.py | 0 ng_openstack/auth.py | 98 +++++++ ng_openstack/cinder.py | 75 +++++ ng_openstack/floatingIP.py | 84 ++++++ ng_openstack/gnocchi.py | 452 +++++++++++++++++++++++++++++++ ng_openstack/keystone.py | 10 + ng_openstack/neutron.py | 11 + ng_openstack/nova.py | 15 + ng_openstack/openstackRequest.py | 37 +++ ng_openstack/settings.py | 3 + requirements.txt | 3 + 16 files changed, 828 insertions(+) create mode 100644 .env.example create mode 100644 .gitignore create mode 100644 README.md create mode 100644 exceptions.py create mode 100644 main.py create mode 100644 ng_openstack/__init__.py create mode 100644 ng_openstack/auth.py create mode 100644 ng_openstack/cinder.py create mode 100644 ng_openstack/floatingIP.py create mode 100644 ng_openstack/gnocchi.py create mode 100644 ng_openstack/keystone.py create mode 100644 ng_openstack/neutron.py create mode 100644 ng_openstack/nova.py create mode 100644 ng_openstack/openstackRequest.py create mode 100644 ng_openstack/settings.py create mode 100644 requirements.txt diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..496197b --- /dev/null +++ b/.env.example @@ -0,0 +1,8 @@ +OS_USERNAME=admin +OS_PASSWORD= +OS_PROJECT_NAME=admin +OS_USER_DOMAIN_NAME=Default +OS_PROJECT_DOMAIN_NAME=Default +OS_AUTH_URL=http://10.10.110.251:5000/v3 +OS_IDENTITY_API_VERSION=3 +OS_INTERFACE=internal diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..95afd81 --- /dev/null +++ b/.gitignore @@ -0,0 +1,12 @@ +.*egg-info.* +build* +dist* +.*python_netflow_v9_softflowd.egg-info/ +*.swp +*.swo +__pycache__ +*.json +venv +.idea +.env +/lib/ \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..c90ba5e --- /dev/null +++ b/README.md @@ -0,0 +1,6 @@ +Run redis localls to store tokens +docker run -d -p 6379:6379 --name redis redis + +sudo apt install python3-dotenv +source venv/bin/activate +pip3 install -r requirements.txt \ No newline at end of file diff --git a/exceptions.py b/exceptions.py new file mode 100644 index 0000000..e80db6a --- /dev/null +++ b/exceptions.py @@ -0,0 +1,7 @@ +class ItemNotFoundError(ValueError): + '''Used when looking up REDIS or SQL or any other database to indicate to the calling routine that the object + cant be found and to move along and try bother some other function + ''' + +class DeadIPFound(ValueError): + ''''An ip has been saved to redis as being 'DEAD"''' diff --git a/main.py b/main.py new file mode 100644 index 0000000..f98d5e9 --- /dev/null +++ b/main.py @@ -0,0 +1,7 @@ +import ng_openstack.auth +import os + +token = ng_openstack.auth.getToken(os.getenv("OS_USERNAME"), os.getenv("OS_PASSWORD"), + os.getenv("OS_USER_DOMAIN_NAME"), os.getenv("OS_USER_DOMAIN_NAME"), + os.getenv("OS_PROJECT_ID")) +print(token) diff --git a/ng_openstack/__init__.py b/ng_openstack/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/ng_openstack/auth.py b/ng_openstack/auth.py new file mode 100644 index 0000000..01d3dc1 --- /dev/null +++ b/ng_openstack/auth.py @@ -0,0 +1,98 @@ +import os +import exceptions +import requests +import json +import redis +import ng_openstack.settings as settings + + +appDebug = True + +def getToken(username, password, authDomain, scopeDomain, scopeProject): + + try: + return lookupTokenFromRedis(username, authDomain, scopeDomain, scopeProject) + except exceptions.ItemNotFoundError: + + url = os.getenv("OS_AUTH_URL") + '/auth/tokens' + xdata = { + "auth": { + "scope": { + "project": { + "domain": { + "name": scopeDomain + }, + "id": scopeProject + } + }, + "identity": { + "password": { + "user": { + "domain": { + "name": authDomain + }, + "password": password, + "name": username + } + }, + "methods": [ + "password" + ] + } + } + } + + response = requests.post(url) + #print(xdata) + headers = {'Content-type': 'application/json'} + #if appDebug: print(url) + + response = requests.post(url, headers=headers, + data=json.dumps(xdata), + verify=False, + timeout=4) + + if response.status_code == 201: + saveTokenToRedis(username, authDomain, scopeDomain, scopeProject, response.headers['X-Subject-Token']) + return response.headers['X-Subject-Token'] + else: + raise ValueError("Error in token response to token request:"+response.text) + + +def lookupTokenFromRedis(username, authDomain, scopeDomain, scopeProject): + + REDIS=redis.Redis() + + REDIS = redis.Redis( + host=os.getenv("REDIS_HOST"), + port=6379, + password="") + RedisKey= "OpenstackToken_" + username + "_" + authDomain + "_" + scopeDomain + "_" + scopeProject + try: + if REDIS.exists(RedisKey): + #print("Got token from redis") + token=REDIS.get(RedisKey).decode("utf-8") + return token + else: + raise exceptions.ItemNotFoundError("OpenstackToken not found in redis") + except: + raise exceptions.ItemNotFoundError("OpenstackToken not found in redis") + +def saveTokenToRedis(username, authDomain, scopeDomain, scopeProject,token): + + REDIS=redis.Redis() + + REDIS = redis.Redis( + host=os.getenv("REDIS_HOST"), + port=6379, + password="") + RedisKey = "OpenstackToken_" + username + "_" + authDomain + "_" + scopeDomain + "_" + scopeProject + Token_CacheTimeout=150 + # print("Saving Token to redis with a {} second timeout".format(Token_CacheTimeout)) + try: + REDIS.set(RedisKey,token) + REDIS.expire(RedisKey, Token_CacheTimeout) + return + except: + raise exceptions.ItemNotFoundError("OpenstackToken not saved in redis") + diff --git a/ng_openstack/cinder.py b/ng_openstack/cinder.py new file mode 100644 index 0000000..8413079 --- /dev/null +++ b/ng_openstack/cinder.py @@ -0,0 +1,75 @@ +import settings,os +import ng_openstack.openstackRequest + +def makeBackup(volumeID,name,description,token): + pass + +def listBackups(projectID,marker="",allBackups=[]): + + + if marker=="": + returnData= ng_openstack.openstackRequest.openstackRequest("GET", "v3/"+os.getenv("OS_PROJECT_ID")+"/backups" + "/detail?all_tenants=True&project_id="+projectID, "", + "http://172.25.110.140:8776").json() + else: + + returnData= ng_openstack.openstackRequest.openstackRequest("GET", "v3/"+os.getenv("OS_PROJECT_ID")+"/backups" + "/detail?all_tenants=True&project_id="+projectID+"&marker="+marker, "", + "http://172.25.110.140:8776").json() + allBackups=allBackups+returnData['backups'] + # print("Got {} records".format(len(returnData['backups']))) + + if len(returnData['backups'])>=1000: + #We hit the limt, make another request + # print("Making another request {} ".format(returnData['backups'][999]['id'])) + nextRequestMarker=returnData['backups'][999]['id'] + # print("Len now is {}".format(len(allBackups))) + allBackups=listBackups(projectID,nextRequestMarker,allBackups) + + + + + return allBackups + +def createBackup(volumeID,backupName,backupDescription,projectID,force=False): + # In order to create a backup for another project as admin you must scope the token to that project, + # if you don't, the backup will be added in the admin project + + data = { + "backup": { + "description": backupDescription, + "name": backupName, + "volume_id": volumeID, + # "incremental": True + "force": force + } + } + return ng_openstack.openstackRequest.openstackRequest("POST", "v3/" + projectID + + "/backups", data, "http://172.25.110.140:8776",projectID).json() + + +def getAllVolumesPerProject(projectID): + return ng_openstack.openstackRequest.openstackRequest("GET","v3/"+os.getenv("OS_PROJECT_ID")+ + "/volumes/detail?all_tenants=True&project_id="+projectID,"","http://172.25.110.140:8776").json() + +def getAllSnapshotsPerProject(projectID): + return ng_openstack.openstackRequest.openstackRequest("GET","v3/"+os.getenv("OS_PROJECT_ID")+ + "/snapshots/detail?all_tenants=True&project_id="+projectID,"","http://172.25.110.140:8776").json() + +def printAllVolumesForProject(projectID): + allVols = getAllVolumesPerProject("5f320f5f187f416794c30772b5fa2a89") + + for volume in allVols['volumes']: + print("{:20} {:10} {:6}".format(volume['name'], volume['size'], volume['volume_type'])) + +def logCurrentVolumeUsagetoGnocchi(projectID): + pass + #Get all volumes for the project + #Split them up into types + #Save to heliostack_meter + +def getBackupSizeOnDisk(volumeID): + # Reach out to MOR + # Run rbd du on the volume ID + # Return the size in bytes + pass \ No newline at end of file diff --git a/ng_openstack/floatingIP.py b/ng_openstack/floatingIP.py new file mode 100644 index 0000000..f8e7f10 --- /dev/null +++ b/ng_openstack/floatingIP.py @@ -0,0 +1,84 @@ +import ng_openstack.settings +import pymysql.cursors +import logging +import exceptions +import os,settings + +SQL_HOST = os.getenv("SQL_HOST") +SQL_USER = os.getenv("SQL_USER") +SQL_PASS = os.getenv("SQL_PASS") +SQL_DB = os.getenv("SQL_DB") + + +def lookupIP_FromSQL(IP): + + SQL = " SELECT project_id,status FROM neutron.floatingips where floating_ip_address=%s" + connection = pymysql.connect(user=SQL_USER, db=SQL_DB, host=SQL_HOST, password=SQL_PASS, + charset='utf8mb4', + cursorclass=pymysql.cursors.DictCursor) + + + with connection.cursor() as cursor: + if cursor.execute(SQL, (IP))>=1: + result = cursor.fetchone() + connection.close() + + returnData = ipLookupResult() + returnData.customerID = result['project_id'] + returnData.bypassLogging = 0 + returnData.IP = IP + return returnData + else: + connection.close() + raise exceptions.ItemNotFoundError("No results in floatingIP database for IP:{}".format(IP)) + +def getCountbyProject_FromSQL(projectID): + + print("Querying Floating IP data from database") + # Connect to the database + connection = pymysql.connect(user=SQL_USER, db=SQL_DB, host=SQL_HOST, password=SQL_PASS, + charset='utf8mb4', + cursorclass=pymysql.cursors.DictCursor) + totalFloatingIPs = 0 + returndata = {} + IPArray = [] + try: + + with connection.cursor() as cursor: + # Read a single record + sql = "SELECT floating_ip_address FROM neutron.floatingips where project_id=%s;" + cursor.execute(sql, (projectID,)) + result = cursor.fetchall() + + for thisItem in result: + IPArray.append(thisItem['floating_ip_address']) + + totalFloatingIPs = len(result) + + finally: + connection.close() + + returndata['total'] = totalFloatingIPs + returndata['floating_ip_address'] = IPArray + return returndata + +def getAllBYOSubnetsfromSQL(): + + sql = "SELECT * , inet_ntoa(network) as readable_network, inet_ntoa(netmask) as readable_netmask, 32 - bit_count(power(2, 32) - netmask - 1) as cidr FROM `networks` where byo=1 ORDER BY `netmask` ASC " + connection = pymysql.connect(user=SQL_USER, db=SQL_DB, host=SQL_HOST, password=SQL_PASS, + charset='utf8mb4', + cursorclass=pymysql.cursors.DictCursor) + + returnData = [] + try: + with connection.cursor() as cursor: + cursor.execute(sql, ()) + result = cursor.fetchall() + for thisItem in result: + returnData.append({"readable_network": thisItem['readable_network'], "readable_netmask": thisItem['readable_netmask'],"int_netmask": thisItem['netmask'],"int_network": thisItem['network'] }) + connection.close() + + return returnData + + except: + connection.close() diff --git a/ng_openstack/gnocchi.py b/ng_openstack/gnocchi.py new file mode 100644 index 0000000..406da68 --- /dev/null +++ b/ng_openstack/gnocchi.py @@ -0,0 +1,452 @@ +import settings,os +import ng_openstack.openstackRequest +import exceptions +import json +import pybill_redis +import requests +import datetime, pytz, calendar + +appDebug = True + + +myAccountingRedisCon=pybill_redis.pyRedis(os.getenv("REDIS_HOST"),6379,"") +PREFIX="pyBill_" + + + + +def add_months(date, months): + months_count = date.month + months + + # Calculate the year + year = date.year + int(months_count / 12) + + # Calculate the month + month = (months_count % 12) + if month == 0: + month = 12 + + # Calculate the day + day = date.day + last_day_of_month = calendar.monthrange(year, month)[1] + if day > last_day_of_month: + day = last_day_of_month + + new_date = datetime.date(year, month, day) + return new_date + +def getBWUsageByMonthforProject(yearNumber, monthNumber, projectID): + fmt = '%Y-%m-%dT%H:%M:%S' + # YY,MM,DD + reportingMonth = datetime.datetime(int(yearNumber), int(monthNumber), 1) + + midnightOnFirstDayofMonthInUTC = datetime.datetime.now(pytz.timezone("UTC")) \ + .replace(year=reportingMonth.year, month=reportingMonth.month, day=1, hour=0, minute=0, second=0, microsecond=0) \ + .astimezone(pytz.utc) + + startTime = midnightOnFirstDayofMonthInUTC.strftime(fmt) + + + # Should we roll to next year? + theNextMonth = add_months(reportingMonth, 1).month + theNextYear = add_months(reportingMonth, 1).year + + midnightOnFirstDayofFollowingMonthInUTC = datetime.datetime.now(pytz.timezone("UTC")) \ + .replace(year=theNextYear, month=theNextMonth, day=1, hour=0, minute=0, second=0, + microsecond=0) \ + .astimezone(pytz.utc) + + endTime = midnightOnFirstDayofFollowingMonthInUTC.strftime(fmt) + + return getUsageDataByProject(projectID,startTime,endTime) + +def getUsageDataByProjectandMetricType(projectID,startTime,endTime,metricType): + + + data = { + "operations": [ + "aggregate", + "sum", + [ + "metric", + metricType, + "sum" + ] + ], + "resource_type": "heliostack_bandwidth_meter", + "search": { + "=": { + "project_id":projectID + } + } + } + + try: + usageData = ng_openstack.openstackRequest.openstackRequest("POST", "aggregates?granularity=3600&start="+startTime+"&stop="+endTime,data, + 'http://' + os.getenv("GNOCCHI_HOST") + ':8041/v1') + # print(usageData.json()) + # print(str(len(usageData.json()['measures']['aggregated'])) + " Entries") + sumTotal = 0 + for thisMeasure in usageData.json()['measures']['aggregated']: + if thisMeasure[2]>1000000000000: + thisMeasure[2]=0 + if thisMeasure[2]<0: + thisMeasure[2]=0 + sumTotal += thisMeasure[2] + + return sumTotal + + except Exception as err: + print("ERROR: Unable to retrieve data for " + metricType + str(err.__dict__)) + sumTotal=0 + return 0 + +def getUsageDataByProject(projectID,startTime,endTime): + print("Inside getUsageDataByProject {} {} {}".format(projectID,startTime,endTime)) + sumTotal=0 + returnObject = {} + + returnObject['DOWNLOAD_PEERING']=getUsageDataByProjectandMetricType(projectID,startTime,endTime,"DOWNLOAD_PEERING") + returnObject['DOWNLOAD_TRANSIT']=getUsageDataByProjectandMetricType(projectID,startTime,endTime,"DOWNLOAD_TRANSIT") + returnObject['UPLOAD_PEERING']=getUsageDataByProjectandMetricType(projectID,startTime,endTime,"UPLOAD_PEERING") + returnObject['UPLOAD_TRANSIT']=getUsageDataByProjectandMetricType(projectID,startTime,endTime,"UPLOAD_TRANSIT") + returnObject['DOWNLOAD_SERVICE']=getUsageDataByProjectandMetricType(projectID,startTime,endTime,"DOWNLOAD_SERVICE") + returnObject['UPLOAD_SERVICE']=getUsageDataByProjectandMetricType(projectID,startTime,endTime,"UPLOAD_SERVICE") + + returnObject['SUM_TOTAL']=returnObject['DOWNLOAD_PEERING']+returnObject['DOWNLOAD_TRANSIT']+returnObject['UPLOAD_PEERING']+returnObject['UPLOAD_TRANSIT'] + + print(returnObject) + return returnObject + +def createMetric(metricName,resourceID): + # resourceID=PREFIX+resourceID + if appDebug:print("Creating metric {} for resourceID {}".format(metricName,resourceID)) + # url='http://'+os.getenv("GNOCCHI_HOST")+':8041/v1/resource/generic/' + resourceID + '/metric' + data = {metricName: { + 'archive_policy_name': "high" + } + } + + # headers = {'Content-type': 'application/json', 'X-Auth-Token': token} + # response = requests.post(url, data=data_json, headers=headers) + response = ng_openstack.openstackRequest.openstackRequest("POST", + 'resource/generic/' + resourceID + '/metric', + data, + 'http://' + os.getenv("GNOCCHI_HOST") + ':8041/v1') + + if response.status_code == 200: + if appDebug:print("Created metric {} for resourceID{}".format(metricName,resourceID)) + for thisMetric in response.json(): + if appDebug:print (thisMetric["name"]) + if thisMetric["name"] == metricName: + return thisMetric["id"] + else: + print ("error in response creating metric " + str(response.status_code)) + print(response.text) + +def findMetric(metricName, resourceID): + # resourceID = PREFIX + resourceID + if appDebug:print("Searching for metric {} in resource {}".format(metricName, resourceID)) + data = {'=': { + 'id': resourceID + } + } + + + # headers = {'Content-type': 'application/json', 'X-Auth-Token': token} + # url='http://'+os.getenv("GNOCCHI_HOST")+':8041/v1/search/resource/generic' + # response = requests.post(url, data=data_json, headers=headers) + response = ng_openstack.openstackRequest.openstackRequest("POST", + 'search/resource/generic', + data, + 'http://' + os.getenv("GNOCCHI_HOST") + ':8041/v1') + if response.status_code == 200: + if len(response.json()) == 1: + foundresource=response.json()[0] + resourceID=foundresource["id"] + + # if 'metrics' not in foundresource: + # raise ValueError("No metrics in this resource") + if metricName in foundresource["metrics"]: + if appDebug: print ("Found existing metric {}".format(foundresource["metrics"][metricName])) + return foundresource["metrics"][metricName] + else: + if appDebug:print ("No metric '%s' found for projectID %s, creating" % (metricName, resourceID)) + return createMetric(metricName,resourceID) + + else: + #Resource does not exist, create it + if appDebug: print ("Resource not found for project ID %s, creating it" % resourceID) + data = { + 'id': resourceID + } + + #data_json = json.dumps(data) + # headers = {'Content-type': 'application/json', 'X-Auth-Token': token} + # url='http://'+os.getenv("GNOCCHI_HOST")+':8041/v1/resource/generic' + # response = requests.post(url, data=data_json, headers=headers) + response = ng_openstack.openstackRequest.openstackRequest("POST", + 'resource/generic', + data, + 'http://' + os.getenv("GNOCCHI_HOST") + ':8041/v1') + + if response.status_code == 201: #201 - Created + #if appDebug: print ("Resource created ok") + + if appDebug: print("Created project resource for project ID: %s" % resourceID) + return findMetric(metricName, resourceID) + + else: + raise ValueError ("Error create new resource "+ str(response.status_code) + response.text) + + + else: + raise ValueError("error in response finding metric" + str(response.status_code) + response.text) + +def checkHeliostack_bandwidth_meterResourceExists(thisFlowMetadata, ipResult): + # Don't prepend the prefix here because it's already been done by the function that calls this function + # resourceID=PREFIX+resourceID + resourceID="" + + #if appDebug: print("Searching for resource for {},{},{} ".format(thisFlowMetadata.billingIP, ipResult.customerID,thisFlowMetadata.routerIP)) + + try: + resourceID=myAccountingRedisCon.lookupResourceExists(thisFlowMetadata, ipResult) + #print("Got redis cache hit for resource {}".format(resourceID)) + return resourceID + + except exceptions.ItemNotFoundError as error: + pass + + data = { + "and": [ + { + "=": { + "project_id": ipResult.customerID + } + }, + { + "=": { + "netFlowSource": thisFlowMetadata.routerIP + } + }, + { + "=": { + "ipAddress": thisFlowMetadata.billingIP + } + } + ] + } + #data_json = json.dumps(data) + + # headers = {'Content-type': 'application/json', 'X-Auth-Token': token} + # url='http://'+os.getenv("GNOCCHI_HOST")+':8041/v1/search/resource/heliostack_bandwidth_meter' + # response = requests.post(url, data=data_json, headers=headers) + response = ng_openstack.openstackRequest.openstackRequest("POST", + 'search/resource/heliostack_bandwidth_meter', + data, + 'http://' + os.getenv("GNOCCHI_HOST") + ':8041/v1') + + + if response.status_code == 200: + if len(response.json()) == 1: + foundresource=response.json()[0] + resourceID=foundresource["id"] + if appDebug:print("Resource found in Gnocchi") + myAccountingRedisCon.saveResourceExists(thisFlowMetadata, ipResult,resourceID) + return resourceID + + else: + # Resource does not exist, create it + resourceID = ipResult.customerID + ":" + \ + thisFlowMetadata.billingIP + ":" + thisFlowMetadata.routerIP + if appDebug: print ("Resource not found for {}, {}, {} ... Creating one".format( + thisFlowMetadata.billingIP, ipResult.customerID,thisFlowMetadata.routerIP)) + + # The value specified below for 'id' will become 'original_resource_id' and at the time of + # writing is never used but needs to be filled so we fill with meaningful data + data = { + "ipAddress": thisFlowMetadata.billingIP, + "netFlowSource": thisFlowMetadata.routerIP, + "id": resourceID, + "project_id": ipResult.customerID + } + + #data_json = json.dumps(data) + # headers = {'Content-type': 'application/json', 'X-Auth-Token': token} + # url='http://'+os.getenv("GNOCCHI_HOST")+':8041/v1/resource/heliostack_bandwidth_meter' + # response = requests.post(url, data=data_json, headers=headers) + response = ng_openstack.openstackRequest.openstackRequest("POST", + 'resource/heliostack_bandwidth_meter', + data, + 'http://' + os.getenv("GNOCCHI_HOST") + ':8041/v1') + + if response.status_code == 201: #201 - Created + print(response.json()) + foundresource = response.json() + resourceID = foundresource["id"] + if appDebug: print ("Resource created ok {}".format(resourceID)) + myAccountingRedisCon.saveResourceExists(thisFlowMetadata, ipResult, resourceID) + return resourceID + + else: + raise ValueError("Error create new resource " + str(response.status_code) + response.text) + + else: + raise ValueError("Error in response finding metric" + str(response.status_code) + response.text) + +def putMeasure(metricID,measureValue,timeStamp): + if appDebug:print ("Putting measure into metricID:"+metricID) + data = [{ + "timestamp": timeStamp, + "value": measureValue + }] + + #data_json = json.dumps(data) + print(json.dumps(data)) + # headers = {'Content-type': 'application/json', 'X-Auth-Token': token} + # url='http://'+os.getenv("GNOCCHI_HOST")+':8041/v1/metric/' + metricID + '/measures' + # response = requests.post(url, data=data_json, headers=headers) + response = ng_openstack.openstackRequest.openstackRequest("POST", + 'metric/' + metricID + '/measures', + data, 'http://' + os.getenv("GNOCCHI_HOST") + ':8041/v1') + + if response.status_code == 202: + if appDebug: print("Added data to metric metricID:" + metricID) + else: + print(response.__dict__) + raise ValueError ("error in response while putting data" + str(response.status_code)) + +def putBulkMeasure(bulkData): + if appDebug:print ("Putting bulk measure data") + if len(bulkData)<3: + print("********SHORT BULK MEASURE DATA********") + + #data_json = json.dumps(bulkData) + print(json.dumps(bulkData)) + # headers = {'Content-type': 'application/json', 'X-Auth-Token': token} + # url='http://'+os.getenv("GNOCCHI_HOST")+':8041/v1/batch/resources/metrics/measures?create_metrics=true' + + response=ng_openstack.openstackRequest.openstackRequest("POST","batch/resources/metrics/measures?create_metrics=true",bulkData,'http://'+os.getenv("GNOCCHI_HOST")+':8041/v1') + # response = requests.post(url, data=data_json, headers=headers) + + if response.status_code == 202: + print ("Added bulk data") + else: + print(response.__dict__) + raise ValueError ("Error in response while putting bulk data" + str(response.status_code)) + +def getHeliostack_measureResourceID(projectID): + + data = { + { + "=": { + "project_id": projectID + } + } + } + + response = ng_openstack.openstackRequest.openstackRequest("POST", + 'search/resource/heliostack_measure', + data, + 'http://' + os.getenv("GNOCCHI_HOST") + ':8041/v1') + + + if response.status_code == 200: + if len(response.json()) == 1: + foundresource=response.json()[0] + resourceID=foundresource["id"] + if appDebug:print("Resource found in Gnocchi") + myAccountingRedisCon.saveResourceExists(thisFlowMetadata, ipResult,resourceID) + return resourceID + + else: + # Resource does not exist, create it + resourceID = ipResult.customerID + ":" + \ + thisFlowMetadata.billingIP + ":" + thisFlowMetadata.routerIP + if appDebug: print ("Resource not found for {}, {}, {} ... Creating one".format( + thisFlowMetadata.billingIP, ipResult.customerID,thisFlowMetadata.routerIP)) + + # The value specified below for 'id' will become 'original_resource_id' and at the time of + # writing is never used but needs to be filled so we fill with meaningful data + data = { + "ipAddress": thisFlowMetadata.billingIP, + "netFlowSource": thisFlowMetadata.routerIP, + "id": resourceID, + "project_id": ipResult.customerID + } + + #data_json = json.dumps(data) + # headers = {'Content-type': 'application/json', 'X-Auth-Token': token} + # url='http://'+os.getenv("GNOCCHI_HOST")+':8041/v1/resource/heliostack_bandwidth_meter' + # response = requests.post(url, data=data_json, headers=headers) + response = ng_openstack.openstackRequest.openstackRequest("POST", + 'resource/heliostack_bandwidth_meter', + data, + 'http://' + os.getenv("GNOCCHI_HOST") + ':8041/v1') + + if response.status_code == 201: #201 - Created + print(response.json()) + foundresource = response.json() + resourceID = foundresource["id"] + if appDebug: print ("Resource created ok {}".format(resourceID)) + myAccountingRedisCon.saveResourceExists(thisFlowMetadata, ipResult, resourceID) + return resourceID + + else: + raise ValueError("Error create new resource " + str(response.status_code) + response.text) + + else: + raise ValueError("Error in response finding metric" + str(response.status_code) + response.text) + +def checkheliostack_consumable_resourceExists(projectID,projectName): + data = { + "=": { + "project_id": projectID + } + } + + + + response = ng_openstack.openstackRequest.openstackRequest("POST", + 'search/resource/heliostack_consumable_resource', + data, + 'http://' + os.getenv("GNOCCHI_HOST") + ':8041/v1') + + if response.status_code == 200: + if len(response.json()) == 1: + foundresource=response.json()[0] + resourceID=foundresource["id"] + if appDebug:print("Resource found in Gnocchi") + return resourceID + + else: + # Resource does not exist, create it + + if appDebug: print ("Resource not found for project {} ... Creating one".format(projectID)) + + # The value specified below for 'id' will become 'original_resource_id' and at the time of + # writing is never used but needs to be filled so we fill with meaningful data + data = { + "id": projectID, + "project_id": projectID, + "project_name": projectName + } + + response = ng_openstack.openstackRequest.openstackRequest("POST", + 'resource/heliostack_consumable_resource', + data, + 'http://' + os.getenv("GNOCCHI_HOST") + ':8041/v1') + + if response.status_code == 201: #201 - Created + print(response.json()) + foundresource = response.json() + resourceID = foundresource["id"] + if appDebug: print ("Resource created ok {}".format(resourceID)) + return resourceID + + else: + raise ValueError("Error create new resource " + str(response.status_code) + response.text) + + else: + raise ValueError("Error in response finding metric" + str(response.status_code) + response.text) \ No newline at end of file diff --git a/ng_openstack/keystone.py b/ng_openstack/keystone.py new file mode 100644 index 0000000..acbf51d --- /dev/null +++ b/ng_openstack/keystone.py @@ -0,0 +1,10 @@ +import json +import ng_openstack.openstackRequest + +def getAllProjects(): + projectData=ng_openstack.openstackRequest.openstackRequest("GET", "v3/projects", "", + "http://172.25.110.138:5000").json() + return projectData + + + diff --git a/ng_openstack/neutron.py b/ng_openstack/neutron.py new file mode 100644 index 0000000..0b71008 --- /dev/null +++ b/ng_openstack/neutron.py @@ -0,0 +1,11 @@ +import os,settings +import ng_openstack.openstackRequest + +def listAllFloatingIPsByProject(projectID): + allIPs=ng_openstack.openstackRequest.openstackRequest("GET", "floatingips?project_id="+projectID, "", + "http://172.25.110.153:9696/v2.0").json() + return allIPs + +def ipInSubnet(network,netmask,ip): + #Network, netmask and ip must be supplied in integer form + return (ip & netmask) == network diff --git a/ng_openstack/nova.py b/ng_openstack/nova.py new file mode 100644 index 0000000..11919a9 --- /dev/null +++ b/ng_openstack/nova.py @@ -0,0 +1,15 @@ +import settings,os +import ng_openstack.openstackRequest + +def getAllServers(projectID): + return ng_openstack.openstackRequest.openstackRequest("GET", "/servers/detail?all_tenants=True&project_id="+projectID, + "", "http://172.25.110.147:8774/v2.1").json() + + + +def getAllFlavors(): + return ng_openstack.openstackRequest.openstackRequest("GET", "/flavors/detail", + "", "http://172.25.110.147:8774/v2.1").json() + + + diff --git a/ng_openstack/openstackRequest.py b/ng_openstack/openstackRequest.py new file mode 100644 index 0000000..ff5754f --- /dev/null +++ b/ng_openstack/openstackRequest.py @@ -0,0 +1,37 @@ +import requests +import json +import ng_openstack.auth +import settings,os + +def openstackRequest(getPost, url, data, apiEndpoint, scopedProjectID=""): + + # Default the scope project to the value set in .env.example, allow the user to override if required(EG creating backups) + if scopedProjectID== "": + scopedProjectID=os.getenv("SNGD_OPENSTACK_SCOPEPROJECTID") + else: + scopedProjectID=scopedProjectID + + str(url).replace(" ","%20") + + data_json = json.dumps(data) + + token = ng_openstack.auth.getToken(os.getenv("OS_USERNAME"), os.getenv("OS_PASSWORD"), + os.getenv("OS_USER_DOMAIN_NAME"), os.getenv("OS_USER_DOMAIN_NAME"), + scopedProjectID) + # print (token) + headers = {'Content-type': 'application/json', 'X-Auth-Token': token} + + url = apiEndpoint + "/" + url + # print (url) + # print(data_json) + if getPost=="GET": + response = requests.get(url, headers=headers) + elif getPost=="POST": + response = requests.post(url, data=data_json, headers=headers) + else: + raise ValueError("Unknown request type") + + if 200<= response.status_code <= 300 : + return response + else: + raise ValueError ("Error in response return code" + str(response.status_code) + str(response.content)) diff --git a/ng_openstack/settings.py b/ng_openstack/settings.py new file mode 100644 index 0000000..eff8b63 --- /dev/null +++ b/ng_openstack/settings.py @@ -0,0 +1,3 @@ +from dotenv import load_dotenv + +load_dotenv() \ No newline at end of file diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..f114529 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,3 @@ +requests +redis +python-dotenv \ No newline at end of file