framework
This commit is contained in:
commit
60fe08f754
|
|
@ -0,0 +1,8 @@
|
||||||
|
OS_USERNAME=admin
|
||||||
|
OS_PASSWORD=
|
||||||
|
OS_PROJECT_NAME=admin
|
||||||
|
OS_USER_DOMAIN_NAME=Default
|
||||||
|
OS_PROJECT_DOMAIN_NAME=Default
|
||||||
|
OS_AUTH_URL=http://10.10.110.251:5000/v3
|
||||||
|
OS_IDENTITY_API_VERSION=3
|
||||||
|
OS_INTERFACE=internal
|
||||||
|
|
@ -0,0 +1,12 @@
|
||||||
|
.*egg-info.*
|
||||||
|
build*
|
||||||
|
dist*
|
||||||
|
.*python_netflow_v9_softflowd.egg-info/
|
||||||
|
*.swp
|
||||||
|
*.swo
|
||||||
|
__pycache__
|
||||||
|
*.json
|
||||||
|
venv
|
||||||
|
.idea
|
||||||
|
.env
|
||||||
|
/lib/
|
||||||
|
|
@ -0,0 +1,6 @@
|
||||||
|
Run redis localls to store tokens
|
||||||
|
docker run -d -p 6379:6379 --name redis redis
|
||||||
|
|
||||||
|
sudo apt install python3-dotenv
|
||||||
|
source venv/bin/activate
|
||||||
|
pip3 install -r requirements.txt
|
||||||
|
|
@ -0,0 +1,7 @@
|
||||||
|
class ItemNotFoundError(ValueError):
|
||||||
|
'''Used when looking up REDIS or SQL or any other database to indicate to the calling routine that the object
|
||||||
|
cant be found and to move along and try bother some other function
|
||||||
|
'''
|
||||||
|
|
||||||
|
class DeadIPFound(ValueError):
|
||||||
|
''''An ip has been saved to redis as being 'DEAD"'''
|
||||||
|
|
@ -0,0 +1,7 @@
|
||||||
|
import ng_openstack.auth
|
||||||
|
import os
|
||||||
|
|
||||||
|
token = ng_openstack.auth.getToken(os.getenv("OS_USERNAME"), os.getenv("OS_PASSWORD"),
|
||||||
|
os.getenv("OS_USER_DOMAIN_NAME"), os.getenv("OS_USER_DOMAIN_NAME"),
|
||||||
|
os.getenv("OS_PROJECT_ID"))
|
||||||
|
print(token)
|
||||||
|
|
@ -0,0 +1,98 @@
|
||||||
|
import os
|
||||||
|
import exceptions
|
||||||
|
import requests
|
||||||
|
import json
|
||||||
|
import redis
|
||||||
|
import ng_openstack.settings as settings
|
||||||
|
|
||||||
|
|
||||||
|
appDebug = True
|
||||||
|
|
||||||
|
def getToken(username, password, authDomain, scopeDomain, scopeProject):
|
||||||
|
|
||||||
|
try:
|
||||||
|
return lookupTokenFromRedis(username, authDomain, scopeDomain, scopeProject)
|
||||||
|
except exceptions.ItemNotFoundError:
|
||||||
|
|
||||||
|
url = os.getenv("OS_AUTH_URL") + '/auth/tokens'
|
||||||
|
xdata = {
|
||||||
|
"auth": {
|
||||||
|
"scope": {
|
||||||
|
"project": {
|
||||||
|
"domain": {
|
||||||
|
"name": scopeDomain
|
||||||
|
},
|
||||||
|
"id": scopeProject
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"identity": {
|
||||||
|
"password": {
|
||||||
|
"user": {
|
||||||
|
"domain": {
|
||||||
|
"name": authDomain
|
||||||
|
},
|
||||||
|
"password": password,
|
||||||
|
"name": username
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"methods": [
|
||||||
|
"password"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
response = requests.post(url)
|
||||||
|
#print(xdata)
|
||||||
|
headers = {'Content-type': 'application/json'}
|
||||||
|
#if appDebug: print(url)
|
||||||
|
|
||||||
|
response = requests.post(url, headers=headers,
|
||||||
|
data=json.dumps(xdata),
|
||||||
|
verify=False,
|
||||||
|
timeout=4)
|
||||||
|
|
||||||
|
if response.status_code == 201:
|
||||||
|
saveTokenToRedis(username, authDomain, scopeDomain, scopeProject, response.headers['X-Subject-Token'])
|
||||||
|
return response.headers['X-Subject-Token']
|
||||||
|
else:
|
||||||
|
raise ValueError("Error in token response to token request:"+response.text)
|
||||||
|
|
||||||
|
|
||||||
|
def lookupTokenFromRedis(username, authDomain, scopeDomain, scopeProject):
|
||||||
|
|
||||||
|
REDIS=redis.Redis()
|
||||||
|
|
||||||
|
REDIS = redis.Redis(
|
||||||
|
host=os.getenv("REDIS_HOST"),
|
||||||
|
port=6379,
|
||||||
|
password="")
|
||||||
|
RedisKey= "OpenstackToken_" + username + "_" + authDomain + "_" + scopeDomain + "_" + scopeProject
|
||||||
|
try:
|
||||||
|
if REDIS.exists(RedisKey):
|
||||||
|
#print("Got token from redis")
|
||||||
|
token=REDIS.get(RedisKey).decode("utf-8")
|
||||||
|
return token
|
||||||
|
else:
|
||||||
|
raise exceptions.ItemNotFoundError("OpenstackToken not found in redis")
|
||||||
|
except:
|
||||||
|
raise exceptions.ItemNotFoundError("OpenstackToken not found in redis")
|
||||||
|
|
||||||
|
def saveTokenToRedis(username, authDomain, scopeDomain, scopeProject,token):
|
||||||
|
|
||||||
|
REDIS=redis.Redis()
|
||||||
|
|
||||||
|
REDIS = redis.Redis(
|
||||||
|
host=os.getenv("REDIS_HOST"),
|
||||||
|
port=6379,
|
||||||
|
password="")
|
||||||
|
RedisKey = "OpenstackToken_" + username + "_" + authDomain + "_" + scopeDomain + "_" + scopeProject
|
||||||
|
Token_CacheTimeout=150
|
||||||
|
# print("Saving Token to redis with a {} second timeout".format(Token_CacheTimeout))
|
||||||
|
try:
|
||||||
|
REDIS.set(RedisKey,token)
|
||||||
|
REDIS.expire(RedisKey, Token_CacheTimeout)
|
||||||
|
return
|
||||||
|
except:
|
||||||
|
raise exceptions.ItemNotFoundError("OpenstackToken not saved in redis")
|
||||||
|
|
||||||
|
|
@ -0,0 +1,75 @@
|
||||||
|
import settings,os
|
||||||
|
import ng_openstack.openstackRequest
|
||||||
|
|
||||||
|
def makeBackup(volumeID,name,description,token):
|
||||||
|
pass
|
||||||
|
|
||||||
|
def listBackups(projectID,marker="",allBackups=[]):
|
||||||
|
|
||||||
|
|
||||||
|
if marker=="":
|
||||||
|
returnData= ng_openstack.openstackRequest.openstackRequest("GET", "v3/"+os.getenv("OS_PROJECT_ID")+"/backups"
|
||||||
|
"/detail?all_tenants=True&project_id="+projectID, "",
|
||||||
|
"http://172.25.110.140:8776").json()
|
||||||
|
else:
|
||||||
|
|
||||||
|
returnData= ng_openstack.openstackRequest.openstackRequest("GET", "v3/"+os.getenv("OS_PROJECT_ID")+"/backups"
|
||||||
|
"/detail?all_tenants=True&project_id="+projectID+"&marker="+marker, "",
|
||||||
|
"http://172.25.110.140:8776").json()
|
||||||
|
allBackups=allBackups+returnData['backups']
|
||||||
|
# print("Got {} records".format(len(returnData['backups'])))
|
||||||
|
|
||||||
|
if len(returnData['backups'])>=1000:
|
||||||
|
#We hit the limt, make another request
|
||||||
|
# print("Making another request {} ".format(returnData['backups'][999]['id']))
|
||||||
|
nextRequestMarker=returnData['backups'][999]['id']
|
||||||
|
# print("Len now is {}".format(len(allBackups)))
|
||||||
|
allBackups=listBackups(projectID,nextRequestMarker,allBackups)
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
return allBackups
|
||||||
|
|
||||||
|
def createBackup(volumeID,backupName,backupDescription,projectID,force=False):
|
||||||
|
# In order to create a backup for another project as admin you must scope the token to that project,
|
||||||
|
# if you don't, the backup will be added in the admin project
|
||||||
|
|
||||||
|
data = {
|
||||||
|
"backup": {
|
||||||
|
"description": backupDescription,
|
||||||
|
"name": backupName,
|
||||||
|
"volume_id": volumeID,
|
||||||
|
# "incremental": True
|
||||||
|
"force": force
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return ng_openstack.openstackRequest.openstackRequest("POST", "v3/" + projectID +
|
||||||
|
"/backups", data, "http://172.25.110.140:8776",projectID).json()
|
||||||
|
|
||||||
|
|
||||||
|
def getAllVolumesPerProject(projectID):
|
||||||
|
return ng_openstack.openstackRequest.openstackRequest("GET","v3/"+os.getenv("OS_PROJECT_ID")+
|
||||||
|
"/volumes/detail?all_tenants=True&project_id="+projectID,"","http://172.25.110.140:8776").json()
|
||||||
|
|
||||||
|
def getAllSnapshotsPerProject(projectID):
|
||||||
|
return ng_openstack.openstackRequest.openstackRequest("GET","v3/"+os.getenv("OS_PROJECT_ID")+
|
||||||
|
"/snapshots/detail?all_tenants=True&project_id="+projectID,"","http://172.25.110.140:8776").json()
|
||||||
|
|
||||||
|
def printAllVolumesForProject(projectID):
|
||||||
|
allVols = getAllVolumesPerProject("5f320f5f187f416794c30772b5fa2a89")
|
||||||
|
|
||||||
|
for volume in allVols['volumes']:
|
||||||
|
print("{:20} {:10} {:6}".format(volume['name'], volume['size'], volume['volume_type']))
|
||||||
|
|
||||||
|
def logCurrentVolumeUsagetoGnocchi(projectID):
|
||||||
|
pass
|
||||||
|
#Get all volumes for the project
|
||||||
|
#Split them up into types
|
||||||
|
#Save to heliostack_meter
|
||||||
|
|
||||||
|
def getBackupSizeOnDisk(volumeID):
|
||||||
|
# Reach out to MOR
|
||||||
|
# Run rbd du on the volume ID
|
||||||
|
# Return the size in bytes
|
||||||
|
pass
|
||||||
|
|
@ -0,0 +1,84 @@
|
||||||
|
import ng_openstack.settings
|
||||||
|
import pymysql.cursors
|
||||||
|
import logging
|
||||||
|
import exceptions
|
||||||
|
import os,settings
|
||||||
|
|
||||||
|
SQL_HOST = os.getenv("SQL_HOST")
|
||||||
|
SQL_USER = os.getenv("SQL_USER")
|
||||||
|
SQL_PASS = os.getenv("SQL_PASS")
|
||||||
|
SQL_DB = os.getenv("SQL_DB")
|
||||||
|
|
||||||
|
|
||||||
|
def lookupIP_FromSQL(IP):
|
||||||
|
|
||||||
|
SQL = " SELECT project_id,status FROM neutron.floatingips where floating_ip_address=%s"
|
||||||
|
connection = pymysql.connect(user=SQL_USER, db=SQL_DB, host=SQL_HOST, password=SQL_PASS,
|
||||||
|
charset='utf8mb4',
|
||||||
|
cursorclass=pymysql.cursors.DictCursor)
|
||||||
|
|
||||||
|
|
||||||
|
with connection.cursor() as cursor:
|
||||||
|
if cursor.execute(SQL, (IP))>=1:
|
||||||
|
result = cursor.fetchone()
|
||||||
|
connection.close()
|
||||||
|
|
||||||
|
returnData = ipLookupResult()
|
||||||
|
returnData.customerID = result['project_id']
|
||||||
|
returnData.bypassLogging = 0
|
||||||
|
returnData.IP = IP
|
||||||
|
return returnData
|
||||||
|
else:
|
||||||
|
connection.close()
|
||||||
|
raise exceptions.ItemNotFoundError("No results in floatingIP database for IP:{}".format(IP))
|
||||||
|
|
||||||
|
def getCountbyProject_FromSQL(projectID):
|
||||||
|
|
||||||
|
print("Querying Floating IP data from database")
|
||||||
|
# Connect to the database
|
||||||
|
connection = pymysql.connect(user=SQL_USER, db=SQL_DB, host=SQL_HOST, password=SQL_PASS,
|
||||||
|
charset='utf8mb4',
|
||||||
|
cursorclass=pymysql.cursors.DictCursor)
|
||||||
|
totalFloatingIPs = 0
|
||||||
|
returndata = {}
|
||||||
|
IPArray = []
|
||||||
|
try:
|
||||||
|
|
||||||
|
with connection.cursor() as cursor:
|
||||||
|
# Read a single record
|
||||||
|
sql = "SELECT floating_ip_address FROM neutron.floatingips where project_id=%s;"
|
||||||
|
cursor.execute(sql, (projectID,))
|
||||||
|
result = cursor.fetchall()
|
||||||
|
|
||||||
|
for thisItem in result:
|
||||||
|
IPArray.append(thisItem['floating_ip_address'])
|
||||||
|
|
||||||
|
totalFloatingIPs = len(result)
|
||||||
|
|
||||||
|
finally:
|
||||||
|
connection.close()
|
||||||
|
|
||||||
|
returndata['total'] = totalFloatingIPs
|
||||||
|
returndata['floating_ip_address'] = IPArray
|
||||||
|
return returndata
|
||||||
|
|
||||||
|
def getAllBYOSubnetsfromSQL():
|
||||||
|
|
||||||
|
sql = "SELECT * , inet_ntoa(network) as readable_network, inet_ntoa(netmask) as readable_netmask, 32 - bit_count(power(2, 32) - netmask - 1) as cidr FROM `networks` where byo=1 ORDER BY `netmask` ASC "
|
||||||
|
connection = pymysql.connect(user=SQL_USER, db=SQL_DB, host=SQL_HOST, password=SQL_PASS,
|
||||||
|
charset='utf8mb4',
|
||||||
|
cursorclass=pymysql.cursors.DictCursor)
|
||||||
|
|
||||||
|
returnData = []
|
||||||
|
try:
|
||||||
|
with connection.cursor() as cursor:
|
||||||
|
cursor.execute(sql, ())
|
||||||
|
result = cursor.fetchall()
|
||||||
|
for thisItem in result:
|
||||||
|
returnData.append({"readable_network": thisItem['readable_network'], "readable_netmask": thisItem['readable_netmask'],"int_netmask": thisItem['netmask'],"int_network": thisItem['network'] })
|
||||||
|
connection.close()
|
||||||
|
|
||||||
|
return returnData
|
||||||
|
|
||||||
|
except:
|
||||||
|
connection.close()
|
||||||
|
|
@ -0,0 +1,452 @@
|
||||||
|
import settings,os
|
||||||
|
import ng_openstack.openstackRequest
|
||||||
|
import exceptions
|
||||||
|
import json
|
||||||
|
import pybill_redis
|
||||||
|
import requests
|
||||||
|
import datetime, pytz, calendar
|
||||||
|
|
||||||
|
appDebug = True
|
||||||
|
|
||||||
|
|
||||||
|
myAccountingRedisCon=pybill_redis.pyRedis(os.getenv("REDIS_HOST"),6379,"")
|
||||||
|
PREFIX="pyBill_"
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
def add_months(date, months):
|
||||||
|
months_count = date.month + months
|
||||||
|
|
||||||
|
# Calculate the year
|
||||||
|
year = date.year + int(months_count / 12)
|
||||||
|
|
||||||
|
# Calculate the month
|
||||||
|
month = (months_count % 12)
|
||||||
|
if month == 0:
|
||||||
|
month = 12
|
||||||
|
|
||||||
|
# Calculate the day
|
||||||
|
day = date.day
|
||||||
|
last_day_of_month = calendar.monthrange(year, month)[1]
|
||||||
|
if day > last_day_of_month:
|
||||||
|
day = last_day_of_month
|
||||||
|
|
||||||
|
new_date = datetime.date(year, month, day)
|
||||||
|
return new_date
|
||||||
|
|
||||||
|
def getBWUsageByMonthforProject(yearNumber, monthNumber, projectID):
|
||||||
|
fmt = '%Y-%m-%dT%H:%M:%S'
|
||||||
|
# YY,MM,DD
|
||||||
|
reportingMonth = datetime.datetime(int(yearNumber), int(monthNumber), 1)
|
||||||
|
|
||||||
|
midnightOnFirstDayofMonthInUTC = datetime.datetime.now(pytz.timezone("UTC")) \
|
||||||
|
.replace(year=reportingMonth.year, month=reportingMonth.month, day=1, hour=0, minute=0, second=0, microsecond=0) \
|
||||||
|
.astimezone(pytz.utc)
|
||||||
|
|
||||||
|
startTime = midnightOnFirstDayofMonthInUTC.strftime(fmt)
|
||||||
|
|
||||||
|
|
||||||
|
# Should we roll to next year?
|
||||||
|
theNextMonth = add_months(reportingMonth, 1).month
|
||||||
|
theNextYear = add_months(reportingMonth, 1).year
|
||||||
|
|
||||||
|
midnightOnFirstDayofFollowingMonthInUTC = datetime.datetime.now(pytz.timezone("UTC")) \
|
||||||
|
.replace(year=theNextYear, month=theNextMonth, day=1, hour=0, minute=0, second=0,
|
||||||
|
microsecond=0) \
|
||||||
|
.astimezone(pytz.utc)
|
||||||
|
|
||||||
|
endTime = midnightOnFirstDayofFollowingMonthInUTC.strftime(fmt)
|
||||||
|
|
||||||
|
return getUsageDataByProject(projectID,startTime,endTime)
|
||||||
|
|
||||||
|
def getUsageDataByProjectandMetricType(projectID,startTime,endTime,metricType):
|
||||||
|
|
||||||
|
|
||||||
|
data = {
|
||||||
|
"operations": [
|
||||||
|
"aggregate",
|
||||||
|
"sum",
|
||||||
|
[
|
||||||
|
"metric",
|
||||||
|
metricType,
|
||||||
|
"sum"
|
||||||
|
]
|
||||||
|
],
|
||||||
|
"resource_type": "heliostack_bandwidth_meter",
|
||||||
|
"search": {
|
||||||
|
"=": {
|
||||||
|
"project_id":projectID
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
try:
|
||||||
|
usageData = ng_openstack.openstackRequest.openstackRequest("POST", "aggregates?granularity=3600&start="+startTime+"&stop="+endTime,data,
|
||||||
|
'http://' + os.getenv("GNOCCHI_HOST") + ':8041/v1')
|
||||||
|
# print(usageData.json())
|
||||||
|
# print(str(len(usageData.json()['measures']['aggregated'])) + " Entries")
|
||||||
|
sumTotal = 0
|
||||||
|
for thisMeasure in usageData.json()['measures']['aggregated']:
|
||||||
|
if thisMeasure[2]>1000000000000:
|
||||||
|
thisMeasure[2]=0
|
||||||
|
if thisMeasure[2]<0:
|
||||||
|
thisMeasure[2]=0
|
||||||
|
sumTotal += thisMeasure[2]
|
||||||
|
|
||||||
|
return sumTotal
|
||||||
|
|
||||||
|
except Exception as err:
|
||||||
|
print("ERROR: Unable to retrieve data for " + metricType + str(err.__dict__))
|
||||||
|
sumTotal=0
|
||||||
|
return 0
|
||||||
|
|
||||||
|
def getUsageDataByProject(projectID,startTime,endTime):
|
||||||
|
print("Inside getUsageDataByProject {} {} {}".format(projectID,startTime,endTime))
|
||||||
|
sumTotal=0
|
||||||
|
returnObject = {}
|
||||||
|
|
||||||
|
returnObject['DOWNLOAD_PEERING']=getUsageDataByProjectandMetricType(projectID,startTime,endTime,"DOWNLOAD_PEERING")
|
||||||
|
returnObject['DOWNLOAD_TRANSIT']=getUsageDataByProjectandMetricType(projectID,startTime,endTime,"DOWNLOAD_TRANSIT")
|
||||||
|
returnObject['UPLOAD_PEERING']=getUsageDataByProjectandMetricType(projectID,startTime,endTime,"UPLOAD_PEERING")
|
||||||
|
returnObject['UPLOAD_TRANSIT']=getUsageDataByProjectandMetricType(projectID,startTime,endTime,"UPLOAD_TRANSIT")
|
||||||
|
returnObject['DOWNLOAD_SERVICE']=getUsageDataByProjectandMetricType(projectID,startTime,endTime,"DOWNLOAD_SERVICE")
|
||||||
|
returnObject['UPLOAD_SERVICE']=getUsageDataByProjectandMetricType(projectID,startTime,endTime,"UPLOAD_SERVICE")
|
||||||
|
|
||||||
|
returnObject['SUM_TOTAL']=returnObject['DOWNLOAD_PEERING']+returnObject['DOWNLOAD_TRANSIT']+returnObject['UPLOAD_PEERING']+returnObject['UPLOAD_TRANSIT']
|
||||||
|
|
||||||
|
print(returnObject)
|
||||||
|
return returnObject
|
||||||
|
|
||||||
|
def createMetric(metricName,resourceID):
|
||||||
|
# resourceID=PREFIX+resourceID
|
||||||
|
if appDebug:print("Creating metric {} for resourceID {}".format(metricName,resourceID))
|
||||||
|
# url='http://'+os.getenv("GNOCCHI_HOST")+':8041/v1/resource/generic/' + resourceID + '/metric'
|
||||||
|
data = {metricName: {
|
||||||
|
'archive_policy_name': "high"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
# headers = {'Content-type': 'application/json', 'X-Auth-Token': token}
|
||||||
|
# response = requests.post(url, data=data_json, headers=headers)
|
||||||
|
response = ng_openstack.openstackRequest.openstackRequest("POST",
|
||||||
|
'resource/generic/' + resourceID + '/metric',
|
||||||
|
data,
|
||||||
|
'http://' + os.getenv("GNOCCHI_HOST") + ':8041/v1')
|
||||||
|
|
||||||
|
if response.status_code == 200:
|
||||||
|
if appDebug:print("Created metric {} for resourceID{}".format(metricName,resourceID))
|
||||||
|
for thisMetric in response.json():
|
||||||
|
if appDebug:print (thisMetric["name"])
|
||||||
|
if thisMetric["name"] == metricName:
|
||||||
|
return thisMetric["id"]
|
||||||
|
else:
|
||||||
|
print ("error in response creating metric " + str(response.status_code))
|
||||||
|
print(response.text)
|
||||||
|
|
||||||
|
def findMetric(metricName, resourceID):
|
||||||
|
# resourceID = PREFIX + resourceID
|
||||||
|
if appDebug:print("Searching for metric {} in resource {}".format(metricName, resourceID))
|
||||||
|
data = {'=': {
|
||||||
|
'id': resourceID
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
# headers = {'Content-type': 'application/json', 'X-Auth-Token': token}
|
||||||
|
# url='http://'+os.getenv("GNOCCHI_HOST")+':8041/v1/search/resource/generic'
|
||||||
|
# response = requests.post(url, data=data_json, headers=headers)
|
||||||
|
response = ng_openstack.openstackRequest.openstackRequest("POST",
|
||||||
|
'search/resource/generic',
|
||||||
|
data,
|
||||||
|
'http://' + os.getenv("GNOCCHI_HOST") + ':8041/v1')
|
||||||
|
if response.status_code == 200:
|
||||||
|
if len(response.json()) == 1:
|
||||||
|
foundresource=response.json()[0]
|
||||||
|
resourceID=foundresource["id"]
|
||||||
|
|
||||||
|
# if 'metrics' not in foundresource:
|
||||||
|
# raise ValueError("No metrics in this resource")
|
||||||
|
if metricName in foundresource["metrics"]:
|
||||||
|
if appDebug: print ("Found existing metric {}".format(foundresource["metrics"][metricName]))
|
||||||
|
return foundresource["metrics"][metricName]
|
||||||
|
else:
|
||||||
|
if appDebug:print ("No metric '%s' found for projectID %s, creating" % (metricName, resourceID))
|
||||||
|
return createMetric(metricName,resourceID)
|
||||||
|
|
||||||
|
else:
|
||||||
|
#Resource does not exist, create it
|
||||||
|
if appDebug: print ("Resource not found for project ID %s, creating it" % resourceID)
|
||||||
|
data = {
|
||||||
|
'id': resourceID
|
||||||
|
}
|
||||||
|
|
||||||
|
#data_json = json.dumps(data)
|
||||||
|
# headers = {'Content-type': 'application/json', 'X-Auth-Token': token}
|
||||||
|
# url='http://'+os.getenv("GNOCCHI_HOST")+':8041/v1/resource/generic'
|
||||||
|
# response = requests.post(url, data=data_json, headers=headers)
|
||||||
|
response = ng_openstack.openstackRequest.openstackRequest("POST",
|
||||||
|
'resource/generic',
|
||||||
|
data,
|
||||||
|
'http://' + os.getenv("GNOCCHI_HOST") + ':8041/v1')
|
||||||
|
|
||||||
|
if response.status_code == 201: #201 - Created
|
||||||
|
#if appDebug: print ("Resource created ok")
|
||||||
|
|
||||||
|
if appDebug: print("Created project resource for project ID: %s" % resourceID)
|
||||||
|
return findMetric(metricName, resourceID)
|
||||||
|
|
||||||
|
else:
|
||||||
|
raise ValueError ("Error create new resource "+ str(response.status_code) + response.text)
|
||||||
|
|
||||||
|
|
||||||
|
else:
|
||||||
|
raise ValueError("error in response finding metric" + str(response.status_code) + response.text)
|
||||||
|
|
||||||
|
def checkHeliostack_bandwidth_meterResourceExists(thisFlowMetadata, ipResult):
|
||||||
|
# Don't prepend the prefix here because it's already been done by the function that calls this function
|
||||||
|
# resourceID=PREFIX+resourceID
|
||||||
|
resourceID=""
|
||||||
|
|
||||||
|
#if appDebug: print("Searching for resource for {},{},{} ".format(thisFlowMetadata.billingIP, ipResult.customerID,thisFlowMetadata.routerIP))
|
||||||
|
|
||||||
|
try:
|
||||||
|
resourceID=myAccountingRedisCon.lookupResourceExists(thisFlowMetadata, ipResult)
|
||||||
|
#print("Got redis cache hit for resource {}".format(resourceID))
|
||||||
|
return resourceID
|
||||||
|
|
||||||
|
except exceptions.ItemNotFoundError as error:
|
||||||
|
pass
|
||||||
|
|
||||||
|
data = {
|
||||||
|
"and": [
|
||||||
|
{
|
||||||
|
"=": {
|
||||||
|
"project_id": ipResult.customerID
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"=": {
|
||||||
|
"netFlowSource": thisFlowMetadata.routerIP
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"=": {
|
||||||
|
"ipAddress": thisFlowMetadata.billingIP
|
||||||
|
}
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
#data_json = json.dumps(data)
|
||||||
|
|
||||||
|
# headers = {'Content-type': 'application/json', 'X-Auth-Token': token}
|
||||||
|
# url='http://'+os.getenv("GNOCCHI_HOST")+':8041/v1/search/resource/heliostack_bandwidth_meter'
|
||||||
|
# response = requests.post(url, data=data_json, headers=headers)
|
||||||
|
response = ng_openstack.openstackRequest.openstackRequest("POST",
|
||||||
|
'search/resource/heliostack_bandwidth_meter',
|
||||||
|
data,
|
||||||
|
'http://' + os.getenv("GNOCCHI_HOST") + ':8041/v1')
|
||||||
|
|
||||||
|
|
||||||
|
if response.status_code == 200:
|
||||||
|
if len(response.json()) == 1:
|
||||||
|
foundresource=response.json()[0]
|
||||||
|
resourceID=foundresource["id"]
|
||||||
|
if appDebug:print("Resource found in Gnocchi")
|
||||||
|
myAccountingRedisCon.saveResourceExists(thisFlowMetadata, ipResult,resourceID)
|
||||||
|
return resourceID
|
||||||
|
|
||||||
|
else:
|
||||||
|
# Resource does not exist, create it
|
||||||
|
resourceID = ipResult.customerID + ":" + \
|
||||||
|
thisFlowMetadata.billingIP + ":" + thisFlowMetadata.routerIP
|
||||||
|
if appDebug: print ("Resource not found for {}, {}, {} ... Creating one".format(
|
||||||
|
thisFlowMetadata.billingIP, ipResult.customerID,thisFlowMetadata.routerIP))
|
||||||
|
|
||||||
|
# The value specified below for 'id' will become 'original_resource_id' and at the time of
|
||||||
|
# writing is never used but needs to be filled so we fill with meaningful data
|
||||||
|
data = {
|
||||||
|
"ipAddress": thisFlowMetadata.billingIP,
|
||||||
|
"netFlowSource": thisFlowMetadata.routerIP,
|
||||||
|
"id": resourceID,
|
||||||
|
"project_id": ipResult.customerID
|
||||||
|
}
|
||||||
|
|
||||||
|
#data_json = json.dumps(data)
|
||||||
|
# headers = {'Content-type': 'application/json', 'X-Auth-Token': token}
|
||||||
|
# url='http://'+os.getenv("GNOCCHI_HOST")+':8041/v1/resource/heliostack_bandwidth_meter'
|
||||||
|
# response = requests.post(url, data=data_json, headers=headers)
|
||||||
|
response = ng_openstack.openstackRequest.openstackRequest("POST",
|
||||||
|
'resource/heliostack_bandwidth_meter',
|
||||||
|
data,
|
||||||
|
'http://' + os.getenv("GNOCCHI_HOST") + ':8041/v1')
|
||||||
|
|
||||||
|
if response.status_code == 201: #201 - Created
|
||||||
|
print(response.json())
|
||||||
|
foundresource = response.json()
|
||||||
|
resourceID = foundresource["id"]
|
||||||
|
if appDebug: print ("Resource created ok {}".format(resourceID))
|
||||||
|
myAccountingRedisCon.saveResourceExists(thisFlowMetadata, ipResult, resourceID)
|
||||||
|
return resourceID
|
||||||
|
|
||||||
|
else:
|
||||||
|
raise ValueError("Error create new resource " + str(response.status_code) + response.text)
|
||||||
|
|
||||||
|
else:
|
||||||
|
raise ValueError("Error in response finding metric" + str(response.status_code) + response.text)
|
||||||
|
|
||||||
|
def putMeasure(metricID,measureValue,timeStamp):
|
||||||
|
if appDebug:print ("Putting measure into metricID:"+metricID)
|
||||||
|
data = [{
|
||||||
|
"timestamp": timeStamp,
|
||||||
|
"value": measureValue
|
||||||
|
}]
|
||||||
|
|
||||||
|
#data_json = json.dumps(data)
|
||||||
|
print(json.dumps(data))
|
||||||
|
# headers = {'Content-type': 'application/json', 'X-Auth-Token': token}
|
||||||
|
# url='http://'+os.getenv("GNOCCHI_HOST")+':8041/v1/metric/' + metricID + '/measures'
|
||||||
|
# response = requests.post(url, data=data_json, headers=headers)
|
||||||
|
response = ng_openstack.openstackRequest.openstackRequest("POST",
|
||||||
|
'metric/' + metricID + '/measures',
|
||||||
|
data, 'http://' + os.getenv("GNOCCHI_HOST") + ':8041/v1')
|
||||||
|
|
||||||
|
if response.status_code == 202:
|
||||||
|
if appDebug: print("Added data to metric metricID:" + metricID)
|
||||||
|
else:
|
||||||
|
print(response.__dict__)
|
||||||
|
raise ValueError ("error in response while putting data" + str(response.status_code))
|
||||||
|
|
||||||
|
def putBulkMeasure(bulkData):
|
||||||
|
if appDebug:print ("Putting bulk measure data")
|
||||||
|
if len(bulkData)<3:
|
||||||
|
print("********SHORT BULK MEASURE DATA********")
|
||||||
|
|
||||||
|
#data_json = json.dumps(bulkData)
|
||||||
|
print(json.dumps(bulkData))
|
||||||
|
# headers = {'Content-type': 'application/json', 'X-Auth-Token': token}
|
||||||
|
# url='http://'+os.getenv("GNOCCHI_HOST")+':8041/v1/batch/resources/metrics/measures?create_metrics=true'
|
||||||
|
|
||||||
|
response=ng_openstack.openstackRequest.openstackRequest("POST","batch/resources/metrics/measures?create_metrics=true",bulkData,'http://'+os.getenv("GNOCCHI_HOST")+':8041/v1')
|
||||||
|
# response = requests.post(url, data=data_json, headers=headers)
|
||||||
|
|
||||||
|
if response.status_code == 202:
|
||||||
|
print ("Added bulk data")
|
||||||
|
else:
|
||||||
|
print(response.__dict__)
|
||||||
|
raise ValueError ("Error in response while putting bulk data" + str(response.status_code))
|
||||||
|
|
||||||
|
def getHeliostack_measureResourceID(projectID):
|
||||||
|
|
||||||
|
data = {
|
||||||
|
{
|
||||||
|
"=": {
|
||||||
|
"project_id": projectID
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
response = ng_openstack.openstackRequest.openstackRequest("POST",
|
||||||
|
'search/resource/heliostack_measure',
|
||||||
|
data,
|
||||||
|
'http://' + os.getenv("GNOCCHI_HOST") + ':8041/v1')
|
||||||
|
|
||||||
|
|
||||||
|
if response.status_code == 200:
|
||||||
|
if len(response.json()) == 1:
|
||||||
|
foundresource=response.json()[0]
|
||||||
|
resourceID=foundresource["id"]
|
||||||
|
if appDebug:print("Resource found in Gnocchi")
|
||||||
|
myAccountingRedisCon.saveResourceExists(thisFlowMetadata, ipResult,resourceID)
|
||||||
|
return resourceID
|
||||||
|
|
||||||
|
else:
|
||||||
|
# Resource does not exist, create it
|
||||||
|
resourceID = ipResult.customerID + ":" + \
|
||||||
|
thisFlowMetadata.billingIP + ":" + thisFlowMetadata.routerIP
|
||||||
|
if appDebug: print ("Resource not found for {}, {}, {} ... Creating one".format(
|
||||||
|
thisFlowMetadata.billingIP, ipResult.customerID,thisFlowMetadata.routerIP))
|
||||||
|
|
||||||
|
# The value specified below for 'id' will become 'original_resource_id' and at the time of
|
||||||
|
# writing is never used but needs to be filled so we fill with meaningful data
|
||||||
|
data = {
|
||||||
|
"ipAddress": thisFlowMetadata.billingIP,
|
||||||
|
"netFlowSource": thisFlowMetadata.routerIP,
|
||||||
|
"id": resourceID,
|
||||||
|
"project_id": ipResult.customerID
|
||||||
|
}
|
||||||
|
|
||||||
|
#data_json = json.dumps(data)
|
||||||
|
# headers = {'Content-type': 'application/json', 'X-Auth-Token': token}
|
||||||
|
# url='http://'+os.getenv("GNOCCHI_HOST")+':8041/v1/resource/heliostack_bandwidth_meter'
|
||||||
|
# response = requests.post(url, data=data_json, headers=headers)
|
||||||
|
response = ng_openstack.openstackRequest.openstackRequest("POST",
|
||||||
|
'resource/heliostack_bandwidth_meter',
|
||||||
|
data,
|
||||||
|
'http://' + os.getenv("GNOCCHI_HOST") + ':8041/v1')
|
||||||
|
|
||||||
|
if response.status_code == 201: #201 - Created
|
||||||
|
print(response.json())
|
||||||
|
foundresource = response.json()
|
||||||
|
resourceID = foundresource["id"]
|
||||||
|
if appDebug: print ("Resource created ok {}".format(resourceID))
|
||||||
|
myAccountingRedisCon.saveResourceExists(thisFlowMetadata, ipResult, resourceID)
|
||||||
|
return resourceID
|
||||||
|
|
||||||
|
else:
|
||||||
|
raise ValueError("Error create new resource " + str(response.status_code) + response.text)
|
||||||
|
|
||||||
|
else:
|
||||||
|
raise ValueError("Error in response finding metric" + str(response.status_code) + response.text)
|
||||||
|
|
||||||
|
def checkheliostack_consumable_resourceExists(projectID,projectName):
|
||||||
|
data = {
|
||||||
|
"=": {
|
||||||
|
"project_id": projectID
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
response = ng_openstack.openstackRequest.openstackRequest("POST",
|
||||||
|
'search/resource/heliostack_consumable_resource',
|
||||||
|
data,
|
||||||
|
'http://' + os.getenv("GNOCCHI_HOST") + ':8041/v1')
|
||||||
|
|
||||||
|
if response.status_code == 200:
|
||||||
|
if len(response.json()) == 1:
|
||||||
|
foundresource=response.json()[0]
|
||||||
|
resourceID=foundresource["id"]
|
||||||
|
if appDebug:print("Resource found in Gnocchi")
|
||||||
|
return resourceID
|
||||||
|
|
||||||
|
else:
|
||||||
|
# Resource does not exist, create it
|
||||||
|
|
||||||
|
if appDebug: print ("Resource not found for project {} ... Creating one".format(projectID))
|
||||||
|
|
||||||
|
# The value specified below for 'id' will become 'original_resource_id' and at the time of
|
||||||
|
# writing is never used but needs to be filled so we fill with meaningful data
|
||||||
|
data = {
|
||||||
|
"id": projectID,
|
||||||
|
"project_id": projectID,
|
||||||
|
"project_name": projectName
|
||||||
|
}
|
||||||
|
|
||||||
|
response = ng_openstack.openstackRequest.openstackRequest("POST",
|
||||||
|
'resource/heliostack_consumable_resource',
|
||||||
|
data,
|
||||||
|
'http://' + os.getenv("GNOCCHI_HOST") + ':8041/v1')
|
||||||
|
|
||||||
|
if response.status_code == 201: #201 - Created
|
||||||
|
print(response.json())
|
||||||
|
foundresource = response.json()
|
||||||
|
resourceID = foundresource["id"]
|
||||||
|
if appDebug: print ("Resource created ok {}".format(resourceID))
|
||||||
|
return resourceID
|
||||||
|
|
||||||
|
else:
|
||||||
|
raise ValueError("Error create new resource " + str(response.status_code) + response.text)
|
||||||
|
|
||||||
|
else:
|
||||||
|
raise ValueError("Error in response finding metric" + str(response.status_code) + response.text)
|
||||||
|
|
@ -0,0 +1,10 @@
|
||||||
|
import json
|
||||||
|
import ng_openstack.openstackRequest
|
||||||
|
|
||||||
|
def getAllProjects():
|
||||||
|
projectData=ng_openstack.openstackRequest.openstackRequest("GET", "v3/projects", "",
|
||||||
|
"http://172.25.110.138:5000").json()
|
||||||
|
return projectData
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -0,0 +1,11 @@
|
||||||
|
import os,settings
|
||||||
|
import ng_openstack.openstackRequest
|
||||||
|
|
||||||
|
def listAllFloatingIPsByProject(projectID):
|
||||||
|
allIPs=ng_openstack.openstackRequest.openstackRequest("GET", "floatingips?project_id="+projectID, "",
|
||||||
|
"http://172.25.110.153:9696/v2.0").json()
|
||||||
|
return allIPs
|
||||||
|
|
||||||
|
def ipInSubnet(network,netmask,ip):
|
||||||
|
#Network, netmask and ip must be supplied in integer form
|
||||||
|
return (ip & netmask) == network
|
||||||
|
|
@ -0,0 +1,15 @@
|
||||||
|
import settings,os
|
||||||
|
import ng_openstack.openstackRequest
|
||||||
|
|
||||||
|
def getAllServers(projectID):
|
||||||
|
return ng_openstack.openstackRequest.openstackRequest("GET", "/servers/detail?all_tenants=True&project_id="+projectID,
|
||||||
|
"", "http://172.25.110.147:8774/v2.1").json()
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
def getAllFlavors():
|
||||||
|
return ng_openstack.openstackRequest.openstackRequest("GET", "/flavors/detail",
|
||||||
|
"", "http://172.25.110.147:8774/v2.1").json()
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -0,0 +1,37 @@
|
||||||
|
import requests
|
||||||
|
import json
|
||||||
|
import ng_openstack.auth
|
||||||
|
import settings,os
|
||||||
|
|
||||||
|
def openstackRequest(getPost, url, data, apiEndpoint, scopedProjectID=""):
|
||||||
|
|
||||||
|
# Default the scope project to the value set in .env.example, allow the user to override if required(EG creating backups)
|
||||||
|
if scopedProjectID== "":
|
||||||
|
scopedProjectID=os.getenv("SNGD_OPENSTACK_SCOPEPROJECTID")
|
||||||
|
else:
|
||||||
|
scopedProjectID=scopedProjectID
|
||||||
|
|
||||||
|
str(url).replace(" ","%20")
|
||||||
|
|
||||||
|
data_json = json.dumps(data)
|
||||||
|
|
||||||
|
token = ng_openstack.auth.getToken(os.getenv("OS_USERNAME"), os.getenv("OS_PASSWORD"),
|
||||||
|
os.getenv("OS_USER_DOMAIN_NAME"), os.getenv("OS_USER_DOMAIN_NAME"),
|
||||||
|
scopedProjectID)
|
||||||
|
# print (token)
|
||||||
|
headers = {'Content-type': 'application/json', 'X-Auth-Token': token}
|
||||||
|
|
||||||
|
url = apiEndpoint + "/" + url
|
||||||
|
# print (url)
|
||||||
|
# print(data_json)
|
||||||
|
if getPost=="GET":
|
||||||
|
response = requests.get(url, headers=headers)
|
||||||
|
elif getPost=="POST":
|
||||||
|
response = requests.post(url, data=data_json, headers=headers)
|
||||||
|
else:
|
||||||
|
raise ValueError("Unknown request type")
|
||||||
|
|
||||||
|
if 200<= response.status_code <= 300 :
|
||||||
|
return response
|
||||||
|
else:
|
||||||
|
raise ValueError ("Error in response return code" + str(response.status_code) + str(response.content))
|
||||||
|
|
@ -0,0 +1,3 @@
|
||||||
|
from dotenv import load_dotenv
|
||||||
|
|
||||||
|
load_dotenv()
|
||||||
|
|
@ -0,0 +1,3 @@
|
||||||
|
requests
|
||||||
|
redis
|
||||||
|
python-dotenv
|
||||||
Loading…
Reference in New Issue