commit - /dev/null
commit + 49cb994257709fa8d6782d1572da50197870a15d
blob - /dev/null
blob + c60d3d4b6941859ebfc76dc9cecb6a4e0ee1a00a (mode 644)
--- /dev/null
+++ .gitignore
+raw
+*.cnf
+zscaler.txt
blob - /dev/null
blob + 7f7c586c1b364d22badebcb08ee11c19386fa14c (mode 755)
--- /dev/null
+++ Netskope_APIEvents-01.py
+#!/usr/bin/env python3
+
+import json
+import urllib.request
+import argparse
+import collections
+from operator import itemgetter
+
+parser = argparse.ArgumentParser(description="API Call to collect data")
+parser.add_argument("tenant", type=str, help="Tenant Name")
+parser.add_argument("token", type=str, help="Tenat API Token")
+parser.add_argument("-t", "--timeperiod", type=int, default='604800', help="Timeperiod (default: 604800)")
+
+try:
+ args = parser.parse_args()
+ tenant = args.tenant
+ token = args.token
+ timeperiod = args.timeperiod
+
+except argparse.ArgumentError as e:
+ print(str(e))
+
+def print_dict(dict):
+ for key, value in sorted(dict.items(), key = itemgetter(1), reverse = True):
+ print ("{:<35s}{:5d}".format(key, value))
+
+base_url = "https://{}.goskope.com/api/v1/events?token={}&type=page&timeperiod={}".format(tenant, token, timeperiod)
+
+req = urllib.request.Request(base_url)
+with urllib.request.urlopen(req) as response:
+ content = response.read()
+json_content = json.loads(content)
+
+domains = collections.Counter()
+categories = collections.Counter()
+
+for i in range (0, len (json_content['data'])):
+ domain = json_content["data"][i]["domain"]
+ ccl = json_content["data"][i]["ccl"]
+ category = json_content["data"][i]["category"]
+ domains[domain] += 1
+ categories[category][count] += 1
+
+
+#print ("===== Domains =====")
+#print_dict(domains)
+
+
+print ("\n===== Categories =====")
+print_dict(categories)
+
blob - /dev/null
blob + a342ac76532551a2aa348d848a6d21fdc6444558 (mode 755)
--- /dev/null
+++ Netskope_APIEvents-02.py
+#!/usr/bin/env python3
+
+import json
+import urllib.request
+import argparse
+from collections import Counter
+from operator import itemgetter
+
+parser = argparse.ArgumentParser(description="API Call to collect data")
+parser.add_argument("tenant", type=str, help="Tenant Name")
+parser.add_argument("token", type=str, help="Tenat API Token")
+parser.add_argument("-t", "--timeperiod", type=int, default='604800', help="Timeperiod (default: 604800)")
+
+try:
+ args = parser.parse_args()
+ tenant = args.tenant
+ token = args.token
+ timeperiod = args.timeperiod
+
+except argparse.ArgumentError as e:
+ print(str(e))
+
+def print_dict(dict, json_content):
+ for key, value in sorted(dict.items(), key = itemgetter(1), reverse = True):
+ print ("{:<35s}{:5d}".format(key, value))
+
+base_url = "https://{}.goskope.com/api/v1/events?token={}&type=page&timeperiod={}".format(tenant, token, timeperiod)
+req = urllib.request.Request(base_url)
+with urllib.request.urlopen(req) as response:
+ content = response.read()
+json_content = json.loads(content)
+
+domains = Counter(data['domain'] for data in json_content['data'])
+categories = Counter(data['category'] for data in json_content['data'])
+
+print ("==== Domains ===")
+print_dict (domains, json_content)
+print ("\n")
+print ("==== Categories ===")
+print_dict (categories)
blob - /dev/null
blob + 6876e352659008e7f6281b765bbb633f22201c0d (mode 755)
--- /dev/null
+++ Netskope_APIEvents-03.py
+#!/usr/bin/env python3
+#
+# Copyright 2019, Mischa Peters <mischa AT netskope DOT com>, Netskope.
+# Version 1.0 - 20191028
+#
+# Requires:
+# - Python 3.x
+#
+import json
+import urllib.request
+import argparse
+from collections import Counter
+from operator import itemgetter
+
+parser = argparse.ArgumentParser(description="Get all events from Netskope API", epilog="2019 (c) Netskope")
+parser.add_argument("tenant", type=str, help="Tenant Name (eg. ams.eu)")
+parser.add_argument("token", type=str, help="Tenat API Token")
+parser.add_argument("-t", "--timeperiod", type=int, default='604800', help="Timeperiod 3600 | 86400 | 604800 | 2592000 (default: 604800)")
+parser.add_argument("-r", "--rows", type=int, default='0', help="Number of rows (default display all)")
+parser.add_argument("-s", "--show", action='store_true', help="Show category hits")
+
+try:
+ args = parser.parse_args()
+ tenant = args.tenant
+ token = args.token
+ timeperiod = args.timeperiod
+ rows = args.rows
+ show = args.show
+
+except argparse.ArgumentError as e:
+ print(str(e))
+
+base_url = "https://{}.goskope.com/api/v1/events?token={}&type=application&timeperiod={}".format(tenant, token, timeperiod)
+
+req = urllib.request.Request(base_url)
+with urllib.request.urlopen(req) as response:
+ content = response.read()
+json_content = json.loads(content)
+
+domain_count = Counter()
+domain_category = {}
+category_count = Counter()
+rows = None if rows == 0 else rows
+
+for i in range (0, len (json_content['data'])):
+ domain = json_content["data"][i]["domain"]
+ ccl = json_content["data"][i]["ccl"]
+ category = json_content["data"][i]["category"]
+ domain_count[domain] += 1
+ domain_category[domain] = category
+ category_count[category] += 1
+
+top_domains = domain_count.most_common(rows)
+print ("{:<40s}{:>5s} - {}".format("Domain", "Hits", "Category"))
+print ("################################################################################")
+for i in top_domains:
+ print ("{:<40s}{:5d} - {}".format(i[0], i[1], domain_category[i[0]]))
+
+print ("")
+if show:
+ top_categories = category_count.most_common()
+ print ("{:<40s}{:>5s}".format("Category", "Hits"))
+ print ("################################################################################")
+ for i in top_categories:
+ print ("{:<40s}{:5d}".format(i[0], i[1]))
+
blob - /dev/null
blob + 12bf977a865a1304827b16c9e24f8356250eeb33 (mode 755)
--- /dev/null
+++ Netskope_APIEvents-04.py
+#!/usr/bin/env python3
+#
+# Copyright 2019, Mischa Peters <mischa AT netskope DOT com>, Netskope.
+# Version 1.0 - 20191028
+#
+# Requires:
+# - Python 3.x
+#
+import json
+import urllib.request
+import argparse
+import collections
+from operator import itemgetter
+
+parser = argparse.ArgumentParser(description="Get all events from Netskope API", epilog="2019 (c) Netskope")
+parser.add_argument("tenant", type=str, help="Tenant Name (eg. ams.eu)")
+parser.add_argument("token", type=str, help="Tenat API Token")
+parser.add_argument("-t", "--timeperiod", type=int, default='604800', help="Timeperiod 3600 | 86400 | 604800 | 2592000 (default: 604800)")
+parser.add_argument("-r", "--rows", type=int, default='0', help="Number of rows (default display all)")
+parser.add_argument("-s", "--show", action='store_true', help="Show category hits")
+
+try:
+ args = parser.parse_args()
+ tenant = args.tenant
+ token = args.token
+ timeperiod = args.timeperiod
+ rows = args.rows
+ show = args.show
+
+except argparse.ArgumentError as e:
+ print(str(e))
+
+base_url = "https://{}.goskope.com/api/v1/events?token={}&type=page&timeperiod={}".format(tenant, token, timeperiod)
+
+req = urllib.request.Request(base_url)
+with urllib.request.urlopen(req) as response:
+ content = response.read()
+json_content = json.loads(content)
+
+#site = {'data': []}
+site = collections.defaultdict(list);
+
+rows = None if rows == 0 else rows
+
+for i in range (0, len (json_content['data'])):
+ json_site = json_content["data"][i]["site"]
+ json_domain = json_content["data"][i]["domain"]
+
+ if json_domain not in site[json_site]:
+ site[json_site].append(json_domain)
+ #print (json_site, "-", json_domain)
+
+print (site)
+
+for key, value in sorted(site.items(), key = itemgetter(0), reverse = False):
+ print ("{:<35s}".format(key), end="")
+ for i in value:
+ print ("{},".format(i), end="")
+ print ("")
+
+
+#top_domains = domain_count.most_common(rows)
+#print ("{:<40s}{:>5s} - {}".format("Domain", "Hits", "Category"))
+#print ("################################################################################")
+#for i in top_domains:
+ #print ("{:<40s}{:5d} - {}".format(i[0], i[1], domain_category[i[0]]))
+
+
blob - /dev/null
blob + bb0e807054c3ea06f44bffc065fc9093cfc3ce14 (mode 755)
--- /dev/null
+++ Netskope_APIEvents-05.py
+#!/usr/bin/env python3
+#
+# Copyright 2019, Mischa Peters <mischa AT netskope DOT com>, Netskope.
+# Version 1.0 - 20191028
+#
+# Requires:
+# - Python 3.x
+#
+import json
+import urllib.request
+import argparse
+from collections import Counter
+from operator import itemgetter
+
+parser = argparse.ArgumentParser(description="Get all events from Netskope API", epilog="2019 (c) Netskope")
+parser.add_argument("tenant", type=str, help="Tenant Name (eg. ams.eu)")
+parser.add_argument("token", type=str, help="Tenat API Token")
+parser.add_argument("-t", "--timeperiod", type=int, default='86400', help="Timeperiod 3600 | 86400 | 604800 | 2592000 (default: 86400)")
+parser.add_argument("-r", "--rows", type=int, default='0', help="Number of rows (default display all)")
+parser.add_argument("-s", "--show", action='store_true', help="Show category hits")
+parser.add_argument("-d", "--debug", action='store_true', help="debug")
+
+try:
+ args = parser.parse_args()
+ tenant = args.tenant
+ token = args.token
+ timeperiod = args.timeperiod
+ rows = args.rows
+ show = args.show
+ debug = args.debug
+
+except argparse.ArgumentError as e:
+ print(str(e))
+
+domain_count = Counter()
+domain_category = {}
+category_count = Counter()
+rows = None if rows == 0 else rows
+
+def get_json(type):
+ domain = "goskope.com"
+ url = f"https://{tenant}.{domain}/api/v1/events?token={token}&type={type}&timeperiod={timeperiod}"
+ req = urllib.request.Request(url)
+ with urllib.request.urlopen(req) as response:
+ content = response.read()
+ json_data = json.loads(content)
+ if debug: print (json_data)
+ return(json_data)
+
+json_content = get_json("application")
+for i in range (0, len (json_content['data'])):
+ domain = json_content["data"][i]["domain"]
+ ccl = json_content["data"][i]["ccl"]
+ category = json_content["data"][i]["category"]
+ domain_count[domain] += 1
+ domain_category[domain] = category
+ category_count[category] += 1
+
+top_domains = domain_count.most_common(rows)
+print (f"{'Domain':<40s}{'Hits':>5s} - Category")
+print ("################################################################################")
+for i in top_domains:
+ print (f"{i[0]:<40s}{i[1]:5d} - {domain_category[i[0]]}")
+
+print ("")
+if show:
+ top_categories = category_count.most_common()
+ print (f"{'Category':<40s}{'Hits':>5s}")
+ print ("################################################################################")
+ for i in top_categories:
+ print (f"{i[0]:<40s}{i[1]:5d}")
blob - /dev/null
blob + 2f4d2dde470040d7784667a07f93363bcfce5135 (mode 755)
--- /dev/null
+++ Netskope_APIEvents-06.py
+#!/usr/bin/env python3
+#
+# Copyright 2019, Mischa Peters <mischa AT netskope DOT com>, Netskope.
+# Version 1.0 - 20191107
+#
+# Requires:
+# - Python 3.x
+#
+import json
+import urllib.request
+import argparse
+import sys
+from urllib.parse import urlparse
+import re
+
+parser = argparse.ArgumentParser(description="Collect all page events from Netskope API and process domains by category and confidence")
+parser.add_argument("tenant", type=str, help="Tenant Name (eg. ams.eu)")
+parser.add_argument("token", type=str, help="Tenant API Token")
+parser.add_argument("-t", "--timeperiod", type=int, default='86400', help="Timeperiod 3600 | 86400 | 604800 | 2592000 (default: 86400)")
+parser.add_argument("-r", "--records", type=int, default=100, help="# of records (default: 100)")
+parser.add_argument("-v", "--verbose", action='store_true', help="verbose")
+parser.add_argument("-d", "--debug", action='store_true', help="debug")
+
+try:
+ args = parser.parse_args()
+ tenant = args.tenant
+ token = args.token
+ timeperiod = args.timeperiod
+ records = args.records
+ verbose = args.verbose
+ debug = args.debug
+
+except argparse.ArgumentError as e:
+ print(str(e))
+
+cursor_up = '\x1b[1A'
+erase_line = '\x1b[2K'
+cct_list = ["Cloud Storage", "Webmail"]
+ccl_list = ["low", "poor"]
+whitelist = re.compile("bla")
+ioc_list = []
+i = 0
+
+if verbose:
+ print("Using Categories: ", end='', flush=True)
+ print(", ".join(map(str,cct_list)))
+ print("Using Rating: ", end='', flush=True)
+ print(", ".join(map(str,ccl_list)))
+ print(f"Applying Whitelist for: {whitelist.pattern}")
+
+def get_json(type):
+ domain = "goskope.com"
+ url = f"https://{tenant}.{domain}/api/v1/events?token={token}&type={type}&timeperiod={timeperiod}"
+ req = urllib.request.Request(url)
+ with urllib.request.urlopen(req) as response:
+ content = response.read()
+ json_data = json.loads(content)
+ if debug: print (json_data)
+ return(json_data)
+
+print()
+print("Processing...", end='', flush=True)
+json_content = get_json("page")
+sys.stdout.write(cursor_up)
+sys.stdout.write(erase_line)
+print()
+
+if verbose:
+ print(f"{'#':>4} {'Domain':<50s} Confidence")
+ print("#######################################################################")
+
+for index, data in enumerate(json_content['data']):
+ if not "domain" in data:
+ domain = urlparse(data["url"]).netloc
+ else:
+ domain = data["domain"]
+ if whitelist.search(domain):
+ continue
+ if data["category"] in cct_list:
+ if data["ccl"] in ccl_list:
+ if domain not in ioc_list:
+ i += 1
+ if verbose: print(f"{i:>4}) {domain:<50s} {data['ccl']}")
+ ioc_list.append(domain)
+ if i == records:
+ break
+
+if verbose: print()
+print(", ".join(map(str,ioc_list)))
blob - /dev/null
blob + 38dceccdad6ee4f657b0943fd4730f947e2d63ca (mode 755)
--- /dev/null
+++ Netskope_APIEvents-07.py
+#!/usr/bin/env python3
+#
+# Copyright 2019, Mischa Peters <mischa AT netskope DOT com>, Netskope.
+# Version 1.0 - 20191107
+#
+# Permission to use, copy, modify, and distribute this software for any
+# purpose with or without fee is hereby granted, provided that the above
+# copyright notice and this permission notice appear in all copies.
+#
+# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+#
+# Requires:
+# - Python 3.x
+#
+import json
+import urllib.request
+import argparse
+import sys
+import urllib.parse
+import re
+import requests
+
+parser = argparse.ArgumentParser(description="Collect all page events from Netskope API and process domains by category and confidence")
+parser.add_argument("tenant", type=str, help="Tenant Name (eg. ams.eu)")
+parser.add_argument("token", type=str, help="Tenant API Token")
+parser.add_argument("-t", "--timeperiod", type=int, default='86400', help="Timeperiod 3600 | 86400 | 604800 | 2592000 (default: 86400)")
+parser.add_argument("-r", "--records", type=int, default=100, help="# of records (default: 100)")
+parser.add_argument("-v", "--verbose", action='store_true', help="verbose")
+parser.add_argument("-d", "--debug", action='store_true', help="print raw json data")
+
+try:
+ args = parser.parse_args()
+ tenant = args.tenant
+ token = args.token
+ timeperiod = args.timeperiod
+ records = args.records
+ verbose = args.verbose
+ debug = args.debug
+
+except argparse.ArgumentError as e:
+ print(str(e))
+
+cct_list = ["Cloud Storage", "Webmail"]
+ccl_list = ["low", "poor"]
+whitelist = re.compile("yahoo")
+ioc_list = []
+
+if verbose:
+ print("Using Categories: ", end='', flush=True)
+ print(", ".join(map(str,cct_list)))
+ print("Using Rating: ", end='', flush=True)
+ print(", ".join(map(str,ccl_list)))
+ print(f"Applying Whitelist: {whitelist.pattern}")
+ print()
+ print(f"{'#':>4} {'Domain':<50s} Confidence")
+ print("#######################################################################")
+
+def get_json(type):
+ domain = "goskope.com"
+ url = f"https://{tenant}.{domain}/api/v1/events?token={token}&type={type}&timeperiod={timeperiod}"
+ req = urllib.request.Request(url)
+ with urllib.request.urlopen(req) as response:
+ content = response.read()
+ json_data = json.loads(content)
+ if debug: print (json_data)
+ return(json_data)
+
+def parse_json(json_content):
+ i = 0
+ for index, data in enumerate(json_content['data']):
+ if not "domain" in data:
+ domain = urllib.parse.urlparse(data["url"]).netloc
+ else:
+ domain = data["domain"]
+ if whitelist.search(domain):
+ continue
+ if data["category"] in cct_list:
+ if data["ccl"] in ccl_list:
+ if domain not in ioc_list:
+ i += 1
+ if verbose: print(f"{i:>4}) {domain:<50s} {data['ccl']}")
+ ioc_list.append(domain)
+
+ return ioc_list
+ #domain_list = ", ".join(map(str,ioc_list[:records]))
+ #return domain_list
+
+json = get_json("page")
+print(parse_json(json))
blob - /dev/null
blob + c8b33bedd4c8b2c34339d849ade2f3cdfea77790 (mode 755)
--- /dev/null
+++ Netskope_APIEvents-08.py
+#!/usr/bin/env python3
+#
+# Copyright 2019, Mischa Peters <mischa AT netskope DOT com>, Netskope.
+# Version 1.0 - 20191107
+#
+# Permission to use, copy, modify, and distribute this software for any
+# purpose with or without fee is hereby granted, provided that the above
+# copyright notice and this permission notice appear in all copies.
+#
+# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+#
+# Requires:
+# - Python 3.x
+#
+import os
+import sys
+import json
+import time
+import re
+import logging
+import urllib.parse
+import requests
+
+NTSKP_TENANT = 'https://astrazeneca.eu.goskope.com'
+NTSKP_TOKEN = '604d0a3b26ea9b22c3ec42130ebbfa8e'
+NTSKP_PERIOD = '2592000'
+cct_list = ["Cloud Storage", "Webmail"]
+ccl_list = ["low", "poor"]
+whitelist = re.compile("yahoo")
+ioc_list = []
+
+ZS_MAX_DOMAINS = 2
+headers = {'Content-Type': 'application/json', 'Cache-Control': 'no-cache', 'User-Agent': 'Netskope_ZscalerImporter1.0'}
+PROXY=''
+
+logging.basicConfig(level=logging.DEBUG)
+logging = logging.getLogger('zsc')
+
+def ntskp_get_domains(headers):
+ uri = f"{NTSKP_TENANT}/api/v1/events?token={NTSKP_TOKEN}&type=page&timeperiod={NTSKP_PERIOD}"
+ try:
+ r = requests.get(uri, headers=headers, proxies=PROXY)
+ r.raise_for_status()
+ except Exception as e:
+ logging.error('Error: ' + str(e))
+ sys.exit(1)
+ json = r.json()
+ limit = (len(json['data']))
+
+ for item in json['data']:
+ if not "domain" in item:
+ domain = urllib.parse.urlparse(item['url']).netloc
+ else:
+ domain = item['domain']
+ if whitelist.search(domain):
+ continue
+ if item['category'] in cct_list:
+ if item['ccl'] in ccl_list:
+ if domain not in ioc_list:
+ print(f"{domain:<50s} {item['ccl']}")
+ endtime = item['timestamp']
+ ioc_list.append(domain)
+ print(limit)
+ print(endtime)
+ starttime = endtime - (10 * 60)
+ print(ioc_list[:ZS_MAX_DOMAINS])
+ return ioc_list[:ZS_MAX_DOMAINS]
+
+
+ntskp_get_domains(headers)
+
+now = int(time.time() * 1000)
+print(now)
+#print(str(time.ctime(int(time.time()))))
blob - /dev/null
blob + 9e8322d0194f6302d54af869f7889207b96b0675 (mode 755)
--- /dev/null
+++ Netskope_APIEvents-09.py
+#!/usr/bin/env python3
+#
+# Copyright 2019, Mischa Peters <mischa AT netskope DOT com>, Netskope.
+# Version 1.0 - 20191028
+#
+# Collects all the page events, counts all the domain hits and category hits
+#
+# Requires:
+# - Python 3.x
+#
+import json
+import urllib.request
+import argparse
+from collections import Counter
+from operator import itemgetter
+
+parser = argparse.ArgumentParser(description="Get all events from Netskope API", epilog="2019 (c) Netskope")
+parser.add_argument("tenant", type=str, help="Tenant Name (eg. ams.eu)")
+parser.add_argument("token", type=str, help="Tenat API Token")
+parser.add_argument("-t", "--timeperiod", type=int, default='86400', help="Timeperiod 3600 | 86400 | 604800 | 2592000 (default: 86400)")
+parser.add_argument("-r", "--rows", type=int, default='0', help="Number of rows (default display all)")
+parser.add_argument("-s", "--show", action='store_true', help="Show category hits")
+parser.add_argument("-d", "--debug", action='store_true', help="debug")
+
+try:
+ args = parser.parse_args()
+ tenant = args.tenant
+ token = args.token
+ timeperiod = args.timeperiod
+ rows = args.rows
+ show = args.show
+ debug = args.debug
+
+except argparse.ArgumentError as e:
+ print(str(e))
+
+domain_count = Counter()
+domain_category = {}
+domain_ccl = {}
+domain_cci = {}
+category_count = Counter()
+rows = None if rows == 0 else rows
+
+def get_json(type):
+ domain = "goskope.com"
+ url = f"https://{tenant}.{domain}/api/v1/events?token={token}&type={type}&timeperiod={timeperiod}"
+ req = urllib.request.Request(url)
+ with urllib.request.urlopen(req) as response:
+ content = response.read()
+ json_data = json.loads(content)
+ if debug: print (json_data)
+ print(json.dumps(json_data, indent=4, sort_keys=True))
+ return(json_data)
+
+json_content = get_json("page")
+for i in range (0, len (json_content['data'])):
+ domain = json_content["data"][i]["domain"]
+ ccl = json_content["data"][i]["ccl"]
+ category = json_content["data"][i]["category"]
+ #ccl = json_content["data"][i]["ccl"]
+ cci = json_content["data"][i]["cci"]
+ domain_count[domain] += 1
+ domain_category[domain] = category
+ domain_ccl[domain] = ccl
+ domain_cci[domain] = cci
+category_count[category] += 1
+
+top_domains = domain_count.most_common(rows)
+print (f"{'Domain':<40s}{'Hits':>5s} - Category")
+print ("################################################################################")
+for i in top_domains:
+ print (f"{i[0]:<40s}{i[1]:5d} - {domain_category[i[0]]} - {domain_ccl[i[0]]}")
+
+print ("")
+if show:
+ top_categories = category_count.most_common()
+ print (f"{'Category':<40s}{'Hits':>5s}")
+ print ("################################################################################")
+ for i in top_categories:
+ print (f"{i[0]:<40s}{i[1]:5d}")
blob - /dev/null
blob + c21d33f123c15f47c2dc896796cbde0b2717f88c (mode 755)
--- /dev/null
+++ Netskope_APIEvents-10.py
+#!/usr/bin/env python3
+#
+import os
+import sys
+import json
+import time
+import re
+import logging
+import urllib.parse
+import requests
+import configparser
+from datetime import datetime
+
+###############################################
+
+CONFIG_FILE = "/home/mischa/netskope/netskope.cnf"
+if not os.path.isfile(CONFIG_FILE):
+ logging.error(f"The config file {CONFIG_FILE} doesn't exist")
+ sys.exit(1)
+config = configparser.RawConfigParser()
+config.read(CONFIG_FILE)
+NTSKP_TENANT = config.get('netskope', 'NTSKP_TENANT')
+NTSKP_TOKEN = config.get('netskope', 'NTSKP_TOKEN')
+NTSKP_PERIOD = config.get('netskope', 'NTSKP_PERIOD')
+NTSKP_SCORE = config.get('netskope', 'NTSKP_SCORE')
+NTSKP_CATEGORIES = config.get('netskope', 'NTSKP_CATEGORIES')
+NTSKP_CONFIDENCE = config.get('netskope', 'NTSKP_CONFIDENCE')
+PROXY = config.get('general', 'PROXY')
+
+###############################################
+
+# Use a custom user-agent string
+UA_STRING = 'NetskopeAPICollector1.0'
+
+# Set logging.INFO to logging.DEBUG for debug information
+logging.basicConfig(level=logging.INFO)
+logging = logging.getLogger('NetskopeAPICollector')
+
+###############################################
+
+def ntskp_get_domains(headers):
+ skip = 0
+ filename = f"/home/mischa/netskope/api-{datetime.now().strftime('%Y%m%d')}.txt"
+ logging.info(f"File {filename} created")
+ ssl_session = requests.Session()
+ logging.debug(f"{ssl_session}")
+
+ while True:
+ uri = f'{NTSKP_TENANT}/api/v1/events?token={NTSKP_TOKEN}&type=page&timeperiod={NTSKP_PERIOD}&skip={skip}'
+ try:
+ r = ssl_session.get(uri, headers=headers, proxies=PROXY)
+ r.raise_for_status()
+ except Exception as e:
+ logging.error(f'Error: {str(e)}')
+ sys.exit(1)
+ json = r.json()
+ #if json['data']:
+ if 'data' in json:
+ if len(json['data']) <= 5000:
+ skip += 5000
+ filter_file = open(filename, "a")
+ logging.debug(f"File {filename} opened")
+ for item in json['data']:
+ if not 'domain' in item:
+ domain = urllib.parse.urlparse(item['url']).netloc
+ else:
+ domain = item['domain']
+
+ #if NTSKP_SAFELIST.search(domain):
+ #print(domain)
+
+ #if item['ccl'] in NTSKP_CONFIDENCE:
+ utctime = datetime.utcfromtimestamp(item['timestamp']).strftime('%Y-%m-%d %H:%M:%S')
+ filter_file.write(f"{utctime},{domain},{item['cci']},{item['category']},{item['ccl']},{item['user']}\n")
+ filter_file.close()
+ logging.debug(f"File {filename} closed")
+ logging.debug(f"Next request, skip: {skip}")
+ else:
+ logging.info(f"No more data to collect")
+ break
+ else:
+ logging.info(f"No more data to collect")
+ break
+ if skip == 500000:
+ logging.info(f"Reached limit")
+ break
+
+###############################################
+
+request_headers = {'Content-Type': 'application/json', 'Cache-Control': 'no-cache', 'User-Agent': UA_STRING}
+ntskp_get_domains(request_headers)
blob - /dev/null
blob + 20338b519969d2cee848feca23ebf1a6d2373c37 (mode 755)
--- /dev/null
+++ Netskope_APIReport-01.pl
+#!/usr/bin/perl -w
+use strict;
+use warnings;
+use autodie;
+use Config::Tiny;
+use HTTP::Tiny;
+use JSON::PP;
+use Text::CSV;
+use File::Temp;
+use MIME::Lite;
+
+my $CONFIG_FILE = "/home/mischa/netskope/netskope.cnf";
+my $config = Config::Tiny->read($CONFIG_FILE, 'utf8');
+my $NTSKP_TENANT = $config->{netskope}{NTSKP_TENANT};
+my $NTSKP_TOKEN = $config->{netskope}{NTSKP_TOKEN};
+my $NTSKP_REPORTID = $config->{netskope}{NTSKP_REPORTID};
+
+my $uri = "$NTSKP_TENANT/api/v1/reports?token=$NTSKP_TOKEN&op=reportInfo&id=$NTSKP_REPORTID";
+my $response = HTTP::Tiny->new->get($uri);
+my $json = JSON::PP->new->utf8->decode($response->{'content'});
+my $data = $json->{'data'}->{'latestScheduledRunInfo'}->{'widgets'};
+my %files;
+
+for my $widget (@{$data}) {
+ my $tmp_file = File::Temp->new(UNLINK => 0, TEMPLATE => 'tempXXXXX', DIR => '/tmp');
+ $files{$tmp_file} = $widget->{'name'};
+ $uri = "$NTSKP_TENANT/api/v1/reports?token=$NTSKP_TOKEN&op=widgetData&id=$widget->{'id'}";
+ $response = HTTP::Tiny->new->get($uri);
+ open my $fh_out, ">", $tmp_file;
+ print $fh_out $response->{'content'};
+ close $fh_out;
+}
+
+my $out_email = "azblocklist.csv";
+my $out_zscaler = "zscaler.txt";
+open my $fh_email, ">", $out_email;
+open my $fh_zscaler, ">", $out_zscaler;
+
+for my $item (keys %files) {
+ my $count = 0;
+ my $csv = Text::CSV->new({binary => 1, auto_diag => 1});
+ open my $fh, "<", $item;
+ my $header = $csv->getline($fh);
+
+ print "$files{$item}\n";
+ print $fh_email "$files{$item}\n";
+
+ while (my $row = $csv->getline($fh)) {
+ last if ($count == 30);
+ if ($row->[1] =~ m/,/) {
+ my @domains = split "," , $row->[1];
+ for my $domain (@domains) {
+ print "$domain,";
+ print $fh_email "$domain,";
+ print $fh_zscaler "$domain\n";
+ }
+ } else {
+ print "$row->[1],";
+ print $fh_email "$row->[1],";
+ print $fh_zscaler "$row->[1]\n";
+ }
+ $count++;
+ }
+ print "\n";
+ print $fh_email "\n";
+ close $fh;
+ unlink $item;
+}
+close $fh_email;
+close $fh_zscaler;
+
+my $msg = MIME::Lite->new(
+ From => 'mischa@high5.nl',
+ To => 'mischa@netskope.com',
+ Cc => 'mischa@high5.nl',
+ Subject => 'AztraZeneca Netskope Blocklist',
+ Type => 'TEXT',
+ Data => "Domains pushed to Zscaler for blocking\n\n"
+);
+$msg->attach(
+ Type => 'text/csv',
+ Path => $out_email,
+ Filename => $out_email
+);
+$msg->send('smtp','mail.high5.nl', Debug=>0);
+unlink $out_email;
blob - /dev/null
blob + 7019a1cb79d9f4c39950e94329b3e81fa73715b7 (mode 755)
--- /dev/null
+++ Netskope_OPLPUploader-01.sh
+#!/bin/bash
+#
+# Copyright 2019, Mischa Peters <mischa AT netskope DOT com>, Netskope.
+# OPLPUploader.sh - Version 1.0 - 20200113
+#
+# Permission to use, copy, modify, and distribute this software for any
+# purpose with or without fee is hereby granted, provided that the above
+# copyright notice and this permission notice appear in all copies.
+#
+# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+#
+# find ${LOCALDIR} -type f -name "*.csv" -maxdepth 1 | sort -V
+
+HOST="ftp://"
+LOCALDIR="/tmp/files"
+REMOTEDIR="/nslogs/user/upload/custom-ASML_SplunkCurlv1/"
+USER=''
+PASS=''
+GLOB='*.csv'
+LOG="/tmp/script.log"
+
+if [ -d ${LOCALDIR} ]; then
+ cd ${LOCALDIR}
+else
+ echo "$(date "+%Y-%m-%d %T") ${LOCALDIR} doesn't exist" | tee -a ${LOG}
+ exit 1
+fi
+
+lftp ${HOST} <<- UPLOAD
+ user "${USER}" "${PASS}"
+ cd "${REMOTEDIR}"
+ mput -E "${GLOB}"
+UPLOAD
+
+if [ ! $? -eq 0 ]; then
+ echo "$(date "+%Y-%m-%d %T") unable to upload files" | tee -a ${LOG}
+ exit 1
+fi
blob - /dev/null
blob + ec255af4c1b82563cd104e44079b039dd379c161 (mode 755)
--- /dev/null
+++ Netskope_ZScalerImporter-01.py
+#!/usr/bin/env python3
+#
+# Copyright 2019-2020, Mischa Peters <mischa AT netskope DOT com>, Netskope.
+# Netskope_ZScalerImporter.py - Version 2.0 - 20200611
+#
+# Permission to use, copy, modify, and distribute this software for any
+# purpose with or without fee is hereby granted, provided that the above
+# copyright notice and this permission notice appear in all copies.
+#
+# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+#
+# ZScaler integration with Netskope
+#
+import os
+import sys
+import re
+import json
+import csv
+import time
+import logging
+import urllib.parse
+import requests
+import configparser
+
+###############################################
+
+CONFIG_FILE = "/home/mischa/netskope/netskope.cnf"
+if not os.path.isfile(CONFIG_FILE):
+ logging.error(f"The config file {CONFIG_FILE} doesn't exist")
+ sys.exit(1)
+config = configparser.RawConfigParser()
+config.read(CONFIG_FILE)
+NTSKP_TENANT = config.get('netskope', 'NTSKP_TENANT')
+NTSKP_TOKEN = config.get('netskope', 'NTSKP_TOKEN')
+ZS_MAX_DOMAINS = int(config.get('zscaler', 'ZS_MAX_DOMAINS'))
+ZS_BASE_URI = config.get('zscaler', 'ZS_BASE_URI')
+ZS_API_KEY = config.get('zscaler', 'ZS_API_KEY')
+ZS_API_USERNAME = config.get('zscaler', 'ZS_API_USERNAME')
+ZS_API_PASSWORD = config.get('zscaler', 'ZS_API_PASSWORD')
+ZS_CATEGORY_NAME = config.get('zscaler', 'ZS_CATEGORY_NAME')
+ZS_CATEGORY_DESC = config.get('zscaler', 'ZS_CATEGORY_DESC')
+PROXY = config.get('general', 'PROXY')
+
+###############################################
+
+# Use a custom user-agent string
+UA_STRING = 'Netskope_ZScalerImporter1.0'
+
+# Set logging.INFO to logging.DEBUG for debug information
+logging.basicConfig(level=logging.DEBUG)
+logging = logging.getLogger('Netskope_ZScalerImporter')
+
+def ntskp_get_domains():
+ ioc_list = []
+ with open('zscaler.txt') as f:
+ ioc_list = f.read().splitlines()
+ logging.debug(ioc_list[:ZS_MAX_DOMAINS])
+ return ioc_list[:ZS_MAX_DOMAINS]
+
+def zs_auth(headers):
+ # Authenticatie against ZScaler API, fetch and return JSESSIONID
+ now = int(time.time() * 1000)
+ n = str(now)[-6:]
+ r = str(int(n) >> 1).zfill(6)
+ key = ""
+ for i in range(0, len(str(n)), 1):
+ key += ZS_API_KEY[int(str(n)[i])]
+ for j in range(0, len(str(r)), 1):
+ key += ZS_API_KEY[int(str(r)[j])+2]
+
+ uri = f'{ZS_BASE_URI}/authenticatedSession'
+ body = {'apiKey': key,
+ 'username': ZS_API_USERNAME,
+ 'password': ZS_API_PASSWORD,
+ 'timestamp': now}
+ try:
+ r = requests.post(uri, data=json.dumps(body), headers=headers, proxies=PROXY)
+ r.raise_for_status()
+ jsessionid = re.sub(r';.*$', "", r.headers['Set-Cookie'])
+ except Exception as e:
+ logging.error(f'Error: {str(e)}')
+ sys.exit(1)
+ return jsessionid
+
+def zs_get_categories(headers):
+ # Find any existing categories matching ZS_CATEGORY_NAME
+ uri = f'{ZS_BASE_URI}/urlCategories/lite'
+ try:
+ r = requests.get(uri, headers=headers, proxies=PROXY)
+ r.raise_for_status()
+ except Exception as e:
+ logging.error(f'Error: {str(e)}')
+ sys.exit(1)
+ data = r.json()
+ for item in data:
+ if item.get('configuredName') == ZS_CATEGORY_NAME:
+ return item.get('id')
+ return None
+
+def zs_update_categories(headers, domains, id = None):
+ # Update the ZS_CATEGORY_NAME with blocklist from Netskope
+ description = f'{ZS_CATEGORY_DESC}\n\nLast Updated: {str(time.ctime(int(time.time())))}'
+ body = {'configuredName': ZS_CATEGORY_NAME,
+ 'customCategory': 'true',
+ 'superCategory': 'SECURITY',
+ 'urls': domains,
+ 'description': description}
+ try:
+ if id == None:
+ uri = f'{ZS_BASE_URI}/urlCategories'
+ r = requests.post(uri, json=body, headers=headers, proxies=PROXY)
+ else:
+ uri = f'{ZS_BASE_URI}/urlCategories/{str(id)}'
+ r = requests.put(uri, json=body, headers=headers, proxies=PROXY)
+ r.raise_for_status()
+ except Exception as e:
+ logging.error(f'Error: {str(e)}')
+ sys.exit(1)
+ return None
+
+def zs_logout(headers):
+ # Logout from ZScaler
+ uri = f'{ZS_BASE_URI}/authenticatedSession'
+ try:
+ r = requests.delete(uri, headers=headers, proxies=PROXY)
+ r.raise_for_status()
+ except Exception as e:
+ logging.error(f'Error: {str(e)}')
+ sys.exit(1)
+ return None
+
+##############################################
+
+request_headers = {'Content-Type': 'application/json', 'Cache-Control': 'no-cache', 'User-Agent': UA_STRING}
+domains = ntskp_get_domains()
+request_headers['Cookie'] = zs_auth(request_headers)
+zs_update_categories(request_headers, domains, zs_get_categories(request_headers))
+zs_logout(request_headers)
+logging.info(f'Netskope added {str(len(domains))} domains added to ZScaler custom URL category {ZS_CATEGORY_NAME}')
blob - /dev/null
blob + 49e0ce16be535302880f12d77b7a8b3c8efcae2a (mode 755)
--- /dev/null
+++ Netskope_ZScalerImporter-02.pl
+#!/usr/bin/perl
+#
+# Copyright 2020, Mischa Peters <mischa AT netskope DOT com>, Netskope.
+# Netskope_ZScalerImporter.pl - Version 3.0 - 20200615
+#
+# Permission to use, copy, modify, and distribute this software for any
+# purpose with or without fee is hereby granted, provided that the above
+# copyright notice and this permission notice appear in all copies.
+#
+# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+#
+# ZScaler integration with Netskope
+#
+use 5.024;
+use strict;
+use warnings;
+use autodie;
+use Config::Tiny;
+use Time::HiRes qw(gettimeofday);
+use POSIX qw(strftime);
+use HTTP::Tiny;
+use HTTP::CookieJar;
+use JSON::PP;
+use Text::CSV;
+use File::Temp;
+use MIME::Lite;
+
+my $VERBOSE = 1;
+my $DEBUG = 1;
+my $CONFIG_FILE = "/home/mischa/netskope/netskope.cnf";
+my $config = Config::Tiny->read($CONFIG_FILE, 'utf8');
+my $NTSKP_TENANT = $config->{netskope}{NTSKP_TENANT};
+my $NTSKP_TOKEN = $config->{netskope}{NTSKP_TOKEN};
+my $NTSKP_REPORTID = $config->{netskope}{NTSKP_REPORTID};
+my $ZS_MAX_DOMAINS = $config->{zscaler}{ZS_MAX_DOMAINS};
+my $ZS_BASE_URI = $config->{zscaler}{ZS_BASE_URI};
+my $ZS_API_KEY = $config->{zscaler}{ZS_API_KEY};
+my $ZS_API_USERNAME = $config->{zscaler}{ZS_API_USERNAME};
+my $ZS_API_PASSWORD = $config->{zscaler}{ZS_API_PASSWORD};
+my $ZS_CATEGORY_NAME = $config->{zscaler}{ZS_CATEGORY_NAME};
+my $ZS_CATEGORY_DESC = $config->{zscaler}{ZS_CATEGORY_DESC};
+my $PROXY = $config->{general}{PROXY};
+my $FILENAME = $config->{general}{FILENAME};
+my $SMTP = $config->{general}{SMTP};
+my $FROM = $config->{general}{FROM};
+my $TO = $config->{general}{TO};
+my $SUBJECT = $config->{general}{SUBJECT};
+my $TEXT = $config->{general}{TEXT} . "\n\n";
+my %headers = ("Content-Type" => "application/json", "Cache-Control" => "no-cache");
+
+### Netskope ###
+sub mail_csv {
+ my $msg = MIME::Lite->new(From=> $FROM, To => $TO, Subject => $SUBJECT, Type => 'TEXT', Data => $TEXT);
+ $msg->attach( Type => 'text/csv', Path => $FILENAME, Filename => $FILENAME);
+ $msg->send('smtp', $SMTP, Debug=>0);
+ say "SMTP $FROM -> $TO - CSV" if $VERBOSE;
+ unlink $FILENAME;
+}
+
+sub check_return {
+ my ($status, $content, $uri) = @_;
+ if ($status !~ /^2/) {
+ print "URI: $uri\nHTTP RESPONSE: $status\n$content\n";
+ my $msg = MIME::Lite->new(From => $FROM, To => $TO, Subject => 'Error', Type => 'TEXT',
+ Data => "URI: $uri\nHTTP RESPONSE: $status\n$content\n",
+ );
+ $msg->send('smtp', $SMTP, Debug=>0);
+ say "SMTP $FROM -> $TO - ERROR" if $VERBOSE;
+ say "exit 1";
+ exit 1;
+ }
+}
+
+sub netskope {
+ ### Collect widget IDs
+ my $uri = "$NTSKP_TENANT/api/v1/reports?token=$NTSKP_TOKEN&op=reportInfo&id=$NTSKP_REPORTID";
+ my $request = HTTP::Tiny->new('default_headers' => \%headers);
+ my $response = $request->get($uri);
+ check_return($response->{'status'}, $response->{'content'}, $uri);
+ my $json = JSON::PP->new->utf8->decode($response->{'content'});
+ my $data = $json->{'data'}->{'latestScheduledRunInfo'}->{'widgets'};
+ if (!$data) { check_return(404, $response->{'content'}, "No Widget Data"); }
+ my %files;
+
+
+ ### Collect widget data and write to CSV
+ for my $widget (@{$data}) {
+ my $tmp_file = File::Temp->new(UNLINK => 0, TEMPLATE => 'tempXXXXX', DIR => '/tmp');
+ $files{$tmp_file} = $widget->{'name'};
+ $uri = "$NTSKP_TENANT/api/v1/reports?token=$NTSKP_TOKEN&op=widgetData&id=$widget->{'id'}";
+ $response = $request->get($uri);
+ print "\n#DEBUG#\nWidget Name: " . $widget->{'name'} . "\nWidgetID: " . $widget->{'id'} . " (ReportID: $NTSKP_REPORTID)\n" . $response->{'content'} if $DEBUG;
+ check_return($response->{'status'}, $response->{'content'}, $uri);
+
+ open my $fh_out, ">", $tmp_file;
+ print $fh_out $response->{'content'};
+ close $fh_out;
+ }
+
+ ### Process domains from CSV
+ my @blocklist;
+ open my $fh_email_out, ">", $FILENAME;
+
+ for my $csv_file (keys %files) {
+ my $count = 0;
+ my $csv = Text::CSV->new({binary => 1, auto_diag => 1});
+ open my $fh_in, "<", $csv_file;
+ my $header = $csv->getline($fh_in);
+
+ print "\n## Widget Name: $files{$csv_file}\nDomains: " if $VERBOSE;
+ print $fh_email_out "$files{$csv_file}\n";
+
+ while (my $row = $csv->getline($fh_in)) {
+ last if ($count == 30);
+ print "$row->[1]," if $VERBOSE;
+ print $fh_email_out "$row->[1],";
+ push @blocklist, $row->[1];
+ $count++;
+ }
+ print "\n" if $VERBOSE;
+ print $fh_email_out "\n";
+ close $fh_in;
+ unlink $csv_file;
+ }
+ close $fh_email_out;
+ return @blocklist;
+}
+
+### Zscaler ###
+
+sub zscaler {
+ my @domains = @{$_[0]};
+
+ ### Authenticate
+ my $now = int(gettimeofday * 1000);
+ my $n = substr($now, -6);
+ my $r = sprintf "%06d", $n >> 1;
+ my $key;
+ for my $i (0..length($n)-1) {
+ $key .= substr($ZS_API_KEY, substr($n, $i, 1), 1);
+ }
+ for my $i (0..length($r)-1) {
+ $key .= substr($ZS_API_KEY, substr($r, $i, 1) + 2, 1);
+ }
+ my $uri = "$ZS_BASE_URI/authenticatedSession";
+ my $body = JSON::PP->new->encode({apiKey => $key, username => $ZS_API_USERNAME, password => $ZS_API_PASSWORD, timestamp => $now});
+ my $jar = HTTP::CookieJar->new;
+ my $request = HTTP::Tiny->new('default_headers' => \%headers, 'cookie_jar' => $jar);
+ my $response = $request->post($uri, {'content' => $body});
+ check_return($response->{'status'}, $response->{'content'}, $uri);
+
+ ### Get filter list id
+ $uri = "$ZS_BASE_URI/urlCategories/lite";
+ $response = $request->get($uri);
+ check_return($response->{'status'}, $response->{'content'}, $uri);
+
+ my $json = JSON::PP->new->utf8->decode($response->{'content'});
+ my $id;
+ for my $item (@{$json}) {
+ if (exists($item->{'configuredName'})) {
+ if ($item->{'configuredName'} eq $ZS_CATEGORY_NAME) {
+ $id = $item->{'id'};
+ }
+ }
+ }
+
+ ### Push Domains
+ $uri = defined($id) ? "$ZS_BASE_URI/urlCategories/$id" : "$ZS_BASE_URI/urlCategories";
+ my $method = defined($id) ? "put" : "post";
+ my $description = "$ZS_CATEGORY_DESC\n\nLast Updated: " . strftime("%Y-%m-%d %H:%M:%S", localtime);
+ $#domains = $#domains >= $ZS_MAX_DOMAINS ? $ZS_MAX_DOMAINS : $#domains;
+ $body = JSON::PP->new->encode({configuredName => $ZS_CATEGORY_NAME, customCategory => 'true', superCategory => 'SECURITY', urls => \@domains, description => $description});
+ $response = $request->$method($uri, {'content' => $body});
+ check_return($response->{'status'}, $response->{'content'}, $uri);
+
+ ### Delete authenticadSession
+ $uri = "$ZS_BASE_URI/authenticatedSession";
+ $response = $request->delete($uri);
+ check_return($response->{'status'}, $response->{'content'}, $uri);
+}
+
+say "Running..." if $VERBOSE;
+my @domains = netskope();
+zscaler(\@domains);
+mail_csv();
+say "Completed." if $VERBOSE;
blob - /dev/null
blob + 325664c1d52da632aecda18cf71a79c4511fb066 (mode 755)
--- /dev/null
+++ Netskope_ZScalerImporter-03.pl
+#!/usr/bin/perl
+#
+# Copyright 2020, Mischa Peters <mischa AT netskope DOT com>, Netskope.
+# Netskope_ZScalerImporter.pl - Version 3.0 - 20200615
+#
+# Permission to use, copy, modify, and distribute this software for any
+# purpose with or without fee is hereby granted, provided that the above
+# copyright notice and this permission notice appear in all copies.
+#
+# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+#
+# ZScaler integration with Netskope
+#
+use 5.024;
+use strict;
+use warnings;
+use autodie;
+use Config::Tiny;
+use Time::HiRes qw(gettimeofday);
+use POSIX qw(strftime);
+use HTTP::Tiny;
+use HTTP::CookieJar;
+use JSON::PP;
+use Text::CSV;
+use MIME::Lite;
+
+my $VERBOSE = 1;
+my $DEBUG = 0;
+my $CONFIG_FILE = "/home/mischa/netskope/netskope.cnf";
+my $config = Config::Tiny->read($CONFIG_FILE, 'utf8');
+my $NTSKP_TENANT = $config->{netskope}{NTSKP_TENANT};
+my $NTSKP_TOKEN = $config->{netskope}{NTSKP_TOKEN};
+my $NTSKP_REPORTID = $config->{netskope}{NTSKP_REPORTID};
+my $ZS_MAX_DOMAINS = $config->{zscaler}{ZS_MAX_DOMAINS};
+my $ZS_BASE_URI = $config->{zscaler}{ZS_BASE_URI};
+my $ZS_API_KEY = $config->{zscaler}{ZS_API_KEY};
+my $ZS_API_USERNAME = $config->{zscaler}{ZS_API_USERNAME};
+my $ZS_API_PASSWORD = $config->{zscaler}{ZS_API_PASSWORD};
+my $ZS_CATEGORY_NAME = $config->{zscaler}{ZS_CATEGORY_NAME};
+my $ZS_CATEGORY_DESC = $config->{zscaler}{ZS_CATEGORY_DESC};
+my $PROXY = $config->{general}{PROXY};
+my $SMTP = $config->{general}{SMTP};
+my $FROM = $config->{general}{FROM};
+my $TO = $config->{general}{TO};
+my $SUBJECT = $config->{general}{SUBJECT};
+my $TEXT = $config->{general}{TEXT} . "\n\n";
+my %HEADERS = ("Content-Type" => "application/json", "Cache-Control" => "no-cache");
+my $EMAIL_CSV;
+
+### Netskope ###
+sub mail_csv {
+ my $msg = MIME::Lite->new(From=> $FROM, To => $TO, Subject => $SUBJECT, Type => 'TEXT', Data => $TEXT);
+ $msg->attach(Type => 'text/csv', Data => $EMAIL_CSV);
+ $msg->send('smtp', $SMTP, Debug=>0);
+ say "SMTP $FROM -> $TO - CSV" if $VERBOSE;
+}
+
+sub check_return {
+ my ($status, $content, $uri) = @_;
+ if ($status !~ /^2/) {
+ print "URI: $uri\nHTTP RESPONSE: $status\n$content\n";
+ my $msg = MIME::Lite->new(From => $FROM, To => $TO, Subject => 'Error', Type => 'TEXT',
+ Data => "URI: $uri\nHTTP RESPONSE: $status\n$content\n",
+ );
+ $msg->send('smtp', $SMTP, Debug=>0);
+ say "SMTP $FROM -> $TO - ERROR" if $VERBOSE;
+ say "exit 1";
+ exit 1;
+ }
+}
+
+sub netskope {
+ ### Collect widget IDs
+ my $uri = "$NTSKP_TENANT/api/v1/reports?token=$NTSKP_TOKEN&op=reportInfo&id=$NTSKP_REPORTID";
+ my $request = HTTP::Tiny->new('default_headers' => \%HEADERS);
+ my $response = $request->get($uri);
+ check_return($response->{'status'}, $response->{'content'}, $uri);
+
+ my $json = JSON::PP->new->utf8->decode($response->{'content'});
+ my $data = $json->{'data'}->{'latestScheduledRunInfo'}->{'widgets'};
+ if (!$data) { check_return(404, $response->{'content'}, "No Widget Data"); }
+ my %csv_content;
+
+ ### Collect widget data and write to CSV
+ for my $widget (@{$data}) {
+ $uri = "$NTSKP_TENANT/api/v1/reports?token=$NTSKP_TOKEN&op=widgetData&id=$widget->{'id'}";
+ $response = $request->get($uri);
+ print "\n#DEBUG#\nWidget Name: " . $widget->{'name'} . "\nWidgetID: " . $widget->{'id'} . " (ReportID: $NTSKP_REPORTID)\n" . $response->{'content'} if $DEBUG;
+ check_return($response->{'status'}, $response->{'content'}, $uri);
+ $csv_content{$widget->{'name'}} = $response->{'content'};
+ }
+
+ ### Process domains from CSV
+ my @blocklist;
+ for my $widget_name (keys %csv_content) {
+ my $count = 0;
+ my $csv = Text::CSV->new({binary => 1, auto_diag => 1});
+ open my $fh_in, "<", \$csv_content{$widget_name};
+ my $header = $csv->getline($fh_in);
+
+ print "\n## Widget Name: $widget_name\n## Domains: " if $VERBOSE;
+ $EMAIL_CSV .= "$widget_name\n";
+ while (my $row = $csv->getline($fh_in)) {
+ last if ($count == 30);
+ print "$row->[1]," if $VERBOSE;
+ $EMAIL_CSV .= "$row->[1],";
+ push @blocklist, $row->[1];
+ $count++;
+ }
+ print "\n" if $VERBOSE;
+ $EMAIL_CSV .= "\n";
+ }
+ return @blocklist;
+}
+
+### Zscaler ###
+
+sub zscaler {
+ my @domains = @{$_[0]};
+
+ ### Authenticate
+ my $now = int(gettimeofday * 1000);
+ my $n = substr($now, -6);
+ my $r = sprintf "%06d", $n >> 1;
+ my $key;
+ for my $i (0..length($n)-1) {
+ $key .= substr($ZS_API_KEY, substr($n, $i, 1), 1);
+ }
+ for my $i (0..length($r)-1) {
+ $key .= substr($ZS_API_KEY, substr($r, $i, 1) + 2, 1);
+ }
+ my $uri = "$ZS_BASE_URI/authenticatedSession";
+ my $body = JSON::PP->new->encode({apiKey => $key, username => $ZS_API_USERNAME, password => $ZS_API_PASSWORD, timestamp => $now});
+ my $jar = HTTP::CookieJar->new;
+ my $request = HTTP::Tiny->new('default_headers' => \%HEADERS, 'cookie_jar' => $jar);
+ my $response = $request->post($uri, {'content' => $body});
+ check_return($response->{'status'}, $response->{'content'}, $uri);
+
+ ### Get filter list id
+ $uri = "$ZS_BASE_URI/urlCategories/lite";
+ $response = $request->get($uri);
+ check_return($response->{'status'}, $response->{'content'}, $uri);
+
+ my $json = JSON::PP->new->utf8->decode($response->{'content'});
+ my $id;
+ for my $item (@{$json}) {
+ if (exists($item->{'configuredName'})) {
+ if ($item->{'configuredName'} eq $ZS_CATEGORY_NAME) {
+ $id = $item->{'id'};
+ }
+ }
+ }
+
+ ### Push Domains
+ $uri = defined($id) ? "$ZS_BASE_URI/urlCategories/$id" : "$ZS_BASE_URI/urlCategories";
+ my $method = defined($id) ? "put" : "post";
+ my $description = "$ZS_CATEGORY_DESC\n\nLast Updated: " . strftime("%Y-%m-%d %H:%M:%S", localtime);
+ $#domains = $#domains >= $ZS_MAX_DOMAINS ? $ZS_MAX_DOMAINS : $#domains;
+ $body = JSON::PP->new->encode({configuredName => $ZS_CATEGORY_NAME, customCategory => 'true', superCategory => 'SECURITY', urls => \@domains, description => $description});
+ $response = $request->$method($uri, {'content' => $body});
+ check_return($response->{'status'}, $response->{'content'}, $uri);
+
+ ### Delete authenticadSession
+ $uri = "$ZS_BASE_URI/authenticatedSession";
+ $response = $request->delete($uri);
+ check_return($response->{'status'}, $response->{'content'}, $uri);
+}
+
+say "Running..." if $VERBOSE;
+my @domains = netskope();
+zscaler(\@domains);
+mail_csv();
+say "Completed." if $VERBOSE;
blob - /dev/null
blob + 12ba9cd2dfd45b1fb0088e6916cb6564701c4c84 (mode 755)
--- /dev/null
+++ Netskope_ZScalerImporter-04.pl
+#!/usr/bin/perl
+#
+# Copyright 2020, Mischa Peters <mischa AT netskope DOT com>, Netskope.
+# Netskope_ZScalerImporter.pl - Version 3.0 - 20200615
+#
+# Permission to use, copy, modify, and distribute this software for any
+# purpose with or without fee is hereby granted, provided that the above
+# copyright notice and this permission notice appear in all copies.
+#
+# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+#
+# ZScaler integration with Netskope
+#
+use 5.024;
+use strict;
+use warnings;
+use autodie;
+use Config::Tiny;
+use Time::HiRes qw(gettimeofday);
+use POSIX qw(strftime);
+use HTTP::Tiny;
+use HTTP::CookieJar;
+use JSON::PP;
+use Text::CSV;
+use MIME::Lite;
+
+my $LOGMODE = "VERBOSE";
+my $CONFIG_FILE = "/home/mischa/netskope/netskope.cnf";
+my $config = Config::Tiny->read($CONFIG_FILE, 'utf8');
+my $NTSKP_TENANT = $config->{netskope}{NTSKP_TENANT};
+my $NTSKP_TOKEN = $config->{netskope}{NTSKP_TOKEN};
+my $NTSKP_REPORTID = $config->{netskope}{NTSKP_REPORTID};
+my $ZS_MAX_DOMAINS = $config->{zscaler}{ZS_MAX_DOMAINS};
+my $ZS_BASE_URI = $config->{zscaler}{ZS_BASE_URI};
+my $ZS_API_KEY = $config->{zscaler}{ZS_API_KEY};
+my $ZS_API_USERNAME = $config->{zscaler}{ZS_API_USERNAME};
+my $ZS_API_PASSWORD = $config->{zscaler}{ZS_API_PASSWORD};
+my $ZS_CATEGORY_NAME = $config->{zscaler}{ZS_CATEGORY_NAME};
+my $ZS_CATEGORY_DESC = $config->{zscaler}{ZS_CATEGORY_DESC};
+my $PROXY = $config->{general}{PROXY};
+my $SMTP = $config->{general}{SMTP};
+my $FROM = $config->{general}{FROM};
+my $TO = $config->{general}{TO};
+my $SUBJECT = $config->{general}{SUBJECT};
+my $TEXT = $config->{general}{TEXT} . "\n\n";
+my %HEADERS = ("Content-Type" => "application/json", "Cache-Control" => "no-cache");
+my $EMAIL_CSV = "";
+
+### Netskope ###
+sub mail_csv {
+ my $msg = MIME::Lite->new(From=> $FROM, To => $TO, Subject => $SUBJECT, Type => 'TEXT', Data => $TEXT);
+ $msg->attach(Type => 'text/csv', Data => $EMAIL_CSV);
+ $msg->send('smtp', $SMTP, Debug=>0);
+ say "MAIL From: $FROM -> $TO - Attach CSV" if $LOGMODE;
+}
+
+sub check_return {
+ my ($status, $content, $uri) = @_;
+ if ($status =~ /^2/ && $LOGMODE) {
+ print "URI: $uri\nHTTP RESPONSE: $status\n";
+ print "CONTENT:\n$content\n" if ($LOGMODE eq "DEBUG");
+ }
+ if ($status !~ /^2/) {
+ print "URI: $uri\nHTTP RESPONSE: $status\n$content\n";
+ my $msg = MIME::Lite->new(From => $FROM, To => $TO, Subject => 'API Error along the way', Type => 'TEXT',
+ Data => "URI: $uri\nHTTP RESPONSE: $status\n$content\n\n",
+ );
+ $msg->attach(Type => 'text/csv', Data => $EMAIL_CSV);
+ $msg->send('smtp', $SMTP, Debug=>0);
+ say "MAIL From: $FROM -> $TO - Error Notification" if $LOGMODE;
+ say "exit 1";
+ exit 1;
+ }
+}
+
+sub netskope {
+ ### Collect widget IDs
+ my $uri = "$NTSKP_TENANT/api/v1/reports?token=$NTSKP_TOKEN&op=reportInfo&id=$NTSKP_REPORTID";
+ my $request = HTTP::Tiny->new('default_headers' => \%HEADERS);
+ my $response = $request->get($uri);
+ check_return($response->{'status'}, $response->{'content'}, $uri);
+
+ my $json = JSON::PP->new->utf8->decode($response->{'content'});
+ my $data = $json->{'data'}->{'latestScheduledRunInfo'}->{'widgets'};
+ if (!$data) { check_return(404, $response->{'content'}, "No Widget Data"); }
+ my %csv_content;
+
+ ### Collect widget data and write to CSV
+ for my $widget (@{$data}) {
+ $uri = "$NTSKP_TENANT/api/v1/reports?token=$NTSKP_TOKEN&op=widgetData&id=$widget->{'id'}";
+ $response = $request->get($uri);
+ print "\n#DEBUG#\nWidget Name: " . $widget->{'name'} . "\nWidgetID: " . $widget->{'id'} . " (ReportID: $NTSKP_REPORTID)\n" . $response->{'content'} if ($LOGMODE && $LOGMODE eq "DEBUG");
+ check_return($response->{'status'}, $response->{'content'}, $uri);
+ $csv_content{$widget->{'name'}} = $response->{'content'};
+ }
+
+ ### Process domains from CSV
+ my @blocklist;
+ for my $widget_name (keys %csv_content) {
+ my $count = 0;
+ my $csv = Text::CSV->new({binary => 1, auto_diag => 1});
+ open my $fh_in, "<", \$csv_content{$widget_name};
+ my $header = $csv->getline($fh_in);
+
+ print "\n## Widget Name: $widget_name\n## Domains: " if $LOGMODE;
+ $EMAIL_CSV .= "$widget_name\n";
+ while (my $row = $csv->getline($fh_in)) {
+ last if ($count == 30);
+ print "$row->[1]," if $LOGMODE;
+ $EMAIL_CSV .= "$row->[1],";
+ push @blocklist, $row->[1];
+ $count++;
+ }
+ print "\n\n" if $LOGMODE;
+ $EMAIL_CSV .= "\n";
+ }
+ return @blocklist;
+}
+
+### Zscaler ###
+
+sub zscaler {
+ my @domains = @{$_[0]};
+
+ ### Authenticate
+ my $now = int(gettimeofday * 1000);
+ my $n = substr($now, -6);
+ my $r = sprintf "%06d", $n >> 1;
+ my $key;
+ for my $i (0..length($n)-1) {
+ $key .= substr($ZS_API_KEY, substr($n, $i, 1), 1);
+ }
+ for my $i (0..length($r)-1) {
+ $key .= substr($ZS_API_KEY, substr($r, $i, 1) + 2, 1);
+ }
+ my $uri = "$ZS_BASE_URI/authenticatedSession";
+ my $body = JSON::PP->new->encode({apiKey => $key, username => $ZS_API_USERNAME, password => $ZS_API_PASSWORD, timestamp => $now});
+ my $jar = HTTP::CookieJar->new;
+ my $request = HTTP::Tiny->new('default_headers' => \%HEADERS, 'cookie_jar' => $jar);
+ my $response = $request->post($uri, {'content' => $body});
+ check_return($response->{'status'}, $response->{'content'}, $uri);
+
+ ### Get filter list id
+ $uri = "$ZS_BASE_URI/urlCategories/lite";
+ $response = $request->get($uri);
+ check_return($response->{'status'}, $response->{'content'}, $uri);
+
+ my $json = JSON::PP->new->utf8->decode($response->{'content'});
+ my $id;
+ for my $item (@{$json}) {
+ if (exists($item->{'configuredName'})) {
+ if ($item->{'configuredName'} eq $ZS_CATEGORY_NAME) {
+ $id = $item->{'id'};
+ }
+ }
+ }
+
+ ### Push Domains
+ $uri = defined($id) ? "$ZS_BASE_URI/urlCategories/$id" : "$ZS_BASE_URI/urlCategories";
+ my $method = defined($id) ? "put" : "post";
+ my $description = "$ZS_CATEGORY_DESC\n\nLast Updated: " . strftime("%Y-%m-%d %H:%M:%S", localtime);
+ $#domains = $#domains >= $ZS_MAX_DOMAINS ? $ZS_MAX_DOMAINS : $#domains;
+ $body = JSON::PP->new->encode({configuredName => $ZS_CATEGORY_NAME, customCategory => 'true', superCategory => 'SECURITY', urls => \@domains, description => $description});
+ $response = $request->$method($uri, {'content' => $body});
+ check_return($response->{'status'}, $response->{'content'}, $uri);
+
+ ### Delete authenticadSession
+ $uri = "$ZS_BASE_URI/authenticatedSession";
+ $response = $request->delete($uri);
+ check_return($response->{'status'}, $response->{'content'}, $uri);
+}
+
+say "Running in $LOGMODE mode..." if $LOGMODE;
+my @domains = netskope();
+zscaler(\@domains);
+mail_csv();
+say "Completed." if $LOGMODE;
blob - /dev/null
blob + 064f34b877b6ca96a7b29f755e2f74a30276cb52 (mode 755)
--- /dev/null
+++ Netskope_ZScalerImporter-05.pl
+#!/usr/bin/perl
+#
+# Copyright 2020, Mischa Peters <mischa AT netskope DOT com>, Netskope.
+# Netskope_ZScalerImporter.pl - Version 3.0 - 20200615
+#
+# Permission to use, copy, modify, and distribute this software for any
+# purpose with or without fee is hereby granted, provided that the above
+# copyright notice and this permission notice appear in all copies.
+#
+# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+#
+# ZScaler integration with Netskope
+#
+use 5.024;
+use strict;
+use warnings;
+use autodie;
+use Config::Tiny;
+use Time::HiRes qw(gettimeofday);
+use POSIX qw(strftime);
+use HTTP::Tiny;
+use HTTP::CookieJar;
+use JSON::PP;
+use Text::CSV;
+use MIME::Lite;
+
+my $LOGMODE = "DEBUG";
+my $CONFIG_FILE = "/home/mischa/netskope/netskope.cnf";
+my $config = Config::Tiny->read($CONFIG_FILE, 'utf8');
+my $USER_COUNT = $config->{report}{USER_COUNT};
+my $MAX_DOMAIN= $config->{report}{MAX_DOMAIN};
+my $NTSKP_TENANT = $config->{netskope}{NTSKP_TENANT};
+my $NTSKP_TOKEN = $config->{netskope}{NTSKP_TOKEN};
+my $NTSKP_REPORTID = $config->{netskope}{NTSKP_REPORTID};
+my $ZS_MAX_DOMAINS = $config->{zscaler}{ZS_MAX_DOMAINS};
+my $ZS_BASE_URI = $config->{zscaler}{ZS_BASE_URI};
+my $ZS_API_KEY = $config->{zscaler}{ZS_API_KEY};
+my $ZS_API_USERNAME = $config->{zscaler}{ZS_API_USERNAME};
+my $ZS_API_PASSWORD = $config->{zscaler}{ZS_API_PASSWORD};
+my $ZS_CATEGORY_NAME = $config->{zscaler}{ZS_CATEGORY_NAME};
+my $ZS_CATEGORY_DESC = $config->{zscaler}{ZS_CATEGORY_DESC};
+my $PROXY = $config->{general}{PROXY};
+my $SMTP = $config->{general}{SMTP};
+my $FROM = $config->{general}{FROM};
+my $TO = $config->{general}{TO};
+my $SUBJECT = $config->{general}{SUBJECT};
+my $TEXT = $config->{general}{TEXT} . "\n\n";
+my %HEADERS = ("Content-Type" => "application/json", "Cache-Control" => "no-cache");
+my $EMAIL_CSV = "";
+
+### Netskope ###
+sub mail_csv {
+ my $msg = MIME::Lite->new(From=> $FROM, To => $TO, Subject => $SUBJECT, Type => 'TEXT', Data => $TEXT);
+ $msg->attach(Type => 'text/csv', Data => $EMAIL_CSV);
+ $msg->send('smtp', $SMTP, Debug=>0);
+ say "MAIL From: $FROM -> $TO - Attach CSV" if $LOGMODE;
+}
+
+sub _check_return {
+ my ($status, $content, $uri) = @_;
+ if ($status =~ /^2/ && $LOGMODE) {
+ print "URI: $uri\nHTTP RESPONSE: $status\n";
+ print "CONTENT:\n$content\n" if ($LOGMODE eq "DEBUG");
+ }
+ if ($status !~ /^2/) {
+ print "URI: $uri\nHTTP RESPONSE: $status\n$content\n";
+ my $msg = MIME::Lite->new(From => $FROM, To => $TO, Subject => 'API Error along the way', Type => 'TEXT',
+ Data => "URI: $uri\nHTTP RESPONSE: $status\n$content\n\n",
+ );
+ $msg->attach(Type => 'text/csv', Data => $EMAIL_CSV);
+ $msg->send('smtp', $SMTP, Debug=>0);
+ say "MAIL From: $FROM -> $TO - Error Notification" if $LOGMODE;
+ say "exit 1";
+ exit 1;
+ }
+}
+
+sub netskope {
+ ### Collect widget IDs
+ my $uri = "$NTSKP_TENANT/api/v1/reports?token=$NTSKP_TOKEN&op=reportInfo&id=$NTSKP_REPORTID";
+ my $request = HTTP::Tiny->new('default_headers' => \%HEADERS);
+ my $response = $request->get($uri);
+ _check_return($response->{'status'}, $response->{'content'}, $uri);
+
+ my $json = JSON::PP->new->utf8->decode($response->{'content'});
+ my $data = $json->{'data'}->{'latestScheduledRunInfo'}->{'widgets'};
+ if (!$data) { _check_return(404, $response->{'content'}, "No Widget Data"); }
+ my %csv_content;
+
+ ### Collect widget data and write to CSV
+ for my $widget (@{$data}) {
+ $uri = "$NTSKP_TENANT/api/v1/reports?token=$NTSKP_TOKEN&op=widgetData&id=$widget->{'id'}";
+ $response = $request->get($uri);
+ print "\n#DEBUG#\nWidget Name: " . $widget->{'name'} . "\nWidgetID: " . $widget->{'id'} . " (ReportID: $NTSKP_REPORTID)\n" . $response->{'content'} if ($LOGMODE && $LOGMODE eq "DEBUG");
+ _check_return($response->{'status'}, $response->{'content'}, $uri);
+ $csv_content{$widget->{'name'}} = $response->{'content'};
+ }
+
+ ### Process domains from CSV
+ my @blocklist;
+ for my $widget_name (keys %csv_content) {
+ my $count = 0;
+ my $csv = Text::CSV->new({binary => 1, auto_diag => 1});
+ open my $fh_in, "<", \$csv_content{$widget_name};
+ my $header = $csv->getline($fh_in);
+
+ print "\n## Widget Name: $widget_name\n## Domains: " if $LOGMODE;
+ $EMAIL_CSV .= "$widget_name\n";
+ DOMAIN:
+ while (my $row = $csv->getline($fh_in)) {
+ last DOMAIN if ($count == $MAX_DOMAIN);
+ if ($row->[4] < $USER_COUNT) {
+ print "$row->[1]," if ($LOGMODE ne "DEBUG");
+ print "$row->[0] - $row->[1] - $row->[2], $row->[3], $row->[4]\n" if ($LOGMODE eq "DEBUG");
+ $EMAIL_CSV .= "$row->[1],";
+ push @blocklist, $row->[1];
+ $count++;
+ }
+ }
+ print "\n\n" if $LOGMODE;
+ $EMAIL_CSV .= "\n";
+ }
+ return @blocklist;
+}
+
+### Zscaler ###
+
+sub zscaler {
+ my @domains = @{$_[0]};
+
+ ### Authenticate
+ my $now = int(gettimeofday * 1000);
+ my $n = substr($now, -6);
+ my $r = sprintf "%06d", $n >> 1;
+ my $key;
+ for my $i (0..length($n)-1) {
+ $key .= substr($ZS_API_KEY, substr($n, $i, 1), 1);
+ }
+ for my $i (0..length($r)-1) {
+ $key .= substr($ZS_API_KEY, substr($r, $i, 1) + 2, 1);
+ }
+ my $uri = "$ZS_BASE_URI/authenticatedSession";
+ my $body = JSON::PP->new->encode({apiKey => $key, username => $ZS_API_USERNAME, password => $ZS_API_PASSWORD, timestamp => $now});
+ my $jar = HTTP::CookieJar->new;
+ my $request = HTTP::Tiny->new('default_headers' => \%HEADERS, 'cookie_jar' => $jar);
+ my $response = $request->post($uri, {'content' => $body});
+ _check_return($response->{'status'}, $response->{'content'}, $uri);
+
+ ### Get filter list id
+ $uri = "$ZS_BASE_URI/urlCategories/lite";
+ $response = $request->get($uri);
+ _check_return($response->{'status'}, $response->{'content'}, $uri);
+
+ my $json = JSON::PP->new->utf8->decode($response->{'content'});
+ my $id;
+ for my $item (@{$json}) {
+ if (exists($item->{'configuredName'})) {
+ if ($item->{'configuredName'} eq $ZS_CATEGORY_NAME) {
+ $id = $item->{'id'};
+ }
+ }
+ }
+
+ ### Push Domains
+ $uri = defined($id) ? "$ZS_BASE_URI/urlCategories/$id" : "$ZS_BASE_URI/urlCategories";
+ my $method = defined($id) ? "put" : "post";
+ my $description = "$ZS_CATEGORY_DESC\n\nLast Updated: " . strftime("%Y-%m-%d %H:%M:%S", localtime);
+ $#domains = $#domains >= $ZS_MAX_DOMAINS ? $ZS_MAX_DOMAINS : $#domains;
+ $body = JSON::PP->new->encode({configuredName => $ZS_CATEGORY_NAME, customCategory => 'true', superCategory => 'SECURITY', urls => \@domains, description => $description});
+ $response = $request->$method($uri, {'content' => $body});
+ _check_return($response->{'status'}, $response->{'content'}, $uri);
+
+ ### Delete authenticadSession
+ $uri = "$ZS_BASE_URI/authenticatedSession";
+ $response = $request->delete($uri);
+ _check_return($response->{'status'}, $response->{'content'}, $uri);
+}
+
+say "Running in $LOGMODE mode..." if $LOGMODE;
+my @domains = netskope();
+zscaler(\@domains);
+mail_csv();
+say "Completed." if $LOGMODE;
blob - /dev/null
blob + b197625d18c65a3a3fe26ff4b852ff215278ea76 (mode 755)
--- /dev/null
+++ Netskope_ZScalerImporter-06.pl
+#!/usr/bin/perl
+#
+# Copyright 2020, Mischa Peters <mischa AT netskope DOT com>, Netskope.
+# Netskope_ZScalerImporter.pl - Version 3.0 - 20200615
+#
+# Permission to use, copy, modify, and distribute this software for any
+# purpose with or without fee is hereby granted, provided that the above
+# copyright notice and this permission notice appear in all copies.
+#
+# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+#
+# ZScaler integration with Netskope
+#
+use 5.024;
+use strict;
+use warnings;
+use autodie;
+use Config::Tiny;
+use Time::HiRes qw(gettimeofday);
+use POSIX qw(strftime);
+use HTTP::Tiny;
+use HTTP::CookieJar;
+use JSON::PP;
+use Text::CSV;
+use MIME::Lite;
+
+my $LOGMODE = "VERBOSE";
+my $CONFIG_FILE = "/home/mischa/netskope/netskope.cnf";
+my $config = Config::Tiny->read($CONFIG_FILE, 'utf8');
+my $USER_COUNT = $config->{report}{USER_COUNT};
+my $MAX_DOMAIN= $config->{report}{MAX_DOMAIN};
+my $NTSKP_TENANT = $config->{netskope}{NTSKP_TENANT};
+my $NTSKP_TOKEN = $config->{netskope}{NTSKP_TOKEN};
+my $NTSKP_REPORTID = $config->{netskope}{NTSKP_REPORTID};
+my $ZS_MAX_DOMAINS = $config->{zscaler}{ZS_MAX_DOMAINS};
+my $ZS_BASE_URI = $config->{zscaler}{ZS_BASE_URI};
+my $ZS_API_KEY = $config->{zscaler}{ZS_API_KEY};
+my $ZS_API_USERNAME = $config->{zscaler}{ZS_API_USERNAME};
+my $ZS_API_PASSWORD = $config->{zscaler}{ZS_API_PASSWORD};
+my $ZS_CATEGORY_NAME = $config->{zscaler}{ZS_CATEGORY_NAME};
+my $ZS_CATEGORY_DESC = $config->{zscaler}{ZS_CATEGORY_DESC};
+my $PROXY = $config->{general}{PROXY};
+my $SMTP = $config->{general}{SMTP};
+my $FROM = $config->{general}{FROM};
+my $TO = $config->{general}{TO};
+my $SUBJECT = $config->{general}{SUBJECT};
+my $TEXT = $config->{general}{TEXT} . "\n\n";
+my %HEADERS = ("Content-Type" => "application/json", "Cache-Control" => "no-cache");
+my $EMAIL_CSV = "";
+
+### Netskope ###
+sub mail_csv {
+ my $msg = MIME::Lite->new(From=> $FROM, To => $TO, Subject => $SUBJECT, Type => 'TEXT', Data => $TEXT);
+ $msg->attach(Type => 'text/csv', Data => $EMAIL_CSV);
+ $msg->send('smtp', $SMTP, Debug=>0);
+ say "MAIL From: $FROM -> $TO - Attach CSV" if $LOGMODE;
+}
+
+sub _check_return {
+ my ($status, $content, $uri) = @_;
+ if ($status =~ /^2/ && $LOGMODE) {
+ print "URI: $uri\nHTTP RESPONSE: $status\n";
+ print "CONTENT:\n$content\n" if ($LOGMODE eq "DEBUG");
+ }
+ if ($status !~ /^2/) {
+ print "URI: $uri\nHTTP RESPONSE: $status\n$content\n";
+ my $msg = MIME::Lite->new(From => $FROM, To => $TO, Subject => 'API Error along the way', Type => 'TEXT',
+ Data => "URI: $uri\nHTTP RESPONSE: $status\n$content\n\n",
+ );
+ $msg->attach(Type => 'text/csv', Data => $EMAIL_CSV);
+ $msg->send('smtp', $SMTP, Debug=>0);
+ say "MAIL From: $FROM -> $TO - Error Notification" if $LOGMODE;
+ say "exit 1";
+ exit 1;
+ }
+}
+
+sub netskope {
+ ### Collect widget IDs
+ my $uri = "$NTSKP_TENANT/api/v1/reports?token=$NTSKP_TOKEN&op=reportInfo&id=$NTSKP_REPORTID";
+ my $request = HTTP::Tiny->new('default_headers' => \%HEADERS);
+ my $response = $request->get($uri);
+ _check_return($response->{'status'}, $response->{'content'}, $uri);
+
+ my $json = JSON::PP->new->utf8->decode($response->{'content'});
+ my $data = $json->{'data'}->{'latestScheduledRunInfo'}->{'widgets'};
+ if (!$data) { _check_return(404, $response->{'content'}, "No Widget Data"); }
+ my %csv_content;
+
+ ### Collect widget data and write to CSV
+ for my $widget (@{$data}) {
+ $uri = "$NTSKP_TENANT/api/v1/reports?token=$NTSKP_TOKEN&op=widgetData&id=$widget->{'id'}";
+ $response = $request->get($uri);
+ print "\n#DEBUG#\nWidget Name: " . $widget->{'name'} . "\nWidgetID: " . $widget->{'id'} . " (ReportID: $NTSKP_REPORTID)\n" . $response->{'content'} if ($LOGMODE && $LOGMODE eq "DEBUG");
+ _check_return($response->{'status'}, $response->{'content'}, $uri);
+ $csv_content{$widget->{'name'}} = $response->{'content'};
+ }
+
+ ### Process domains from CSV
+ my @blocklist;
+ for my $widget_name (keys %csv_content) {
+ my $count = 0;
+ my $csv = Text::CSV->new({binary => 1, auto_diag => 1});
+ open my $fh_in, "<", \$csv_content{$widget_name};
+ $csv->column_names($csv->getline($fh_in));
+ # "Application","Domain","Category","CCI","Users"
+
+ print "\n## Widget Name: $widget_name\n## Domains: " if $LOGMODE;
+ $EMAIL_CSV .= "$widget_name\n";
+ DOMAIN:
+ while (my $row = $csv->getline_hr($fh_in)) {
+ last DOMAIN if ($count == $MAX_DOMAIN);
+ if ($row->{'Users'} < $USER_COUNT) {
+ print "$row->{'Domain'}," if ($LOGMODE ne "DEBUG");
+ print "$row->{'Application'} - $row->{'Domain'} - $row->{'Category'}, $row->{'CCI'}, $row->{'Users'}\n" if ($LOGMODE eq "DEBUG");
+ $EMAIL_CSV .= "$row->{'Domain'},";
+ push @blocklist, $row->{'Domain'};
+ $count++;
+ }
+ }
+ print "\n\n" if $LOGMODE;
+ $EMAIL_CSV .= "\n";
+ }
+ return @blocklist;
+}
+
+### Zscaler ###
+
+sub zscaler {
+ my @domains = @{$_[0]};
+
+ ### Authenticate
+ my $now = int(gettimeofday * 1000);
+ my $n = substr($now, -6);
+ my $r = sprintf "%06d", $n >> 1;
+ my $key;
+ for my $i (0..length($n)-1) {
+ $key .= substr($ZS_API_KEY, substr($n, $i, 1), 1);
+ }
+ for my $i (0..length($r)-1) {
+ $key .= substr($ZS_API_KEY, substr($r, $i, 1) + 2, 1);
+ }
+ my $uri = "$ZS_BASE_URI/authenticatedSession";
+ my $body = JSON::PP->new->encode({apiKey => $key, username => $ZS_API_USERNAME, password => $ZS_API_PASSWORD, timestamp => $now});
+ my $jar = HTTP::CookieJar->new;
+ my $request = HTTP::Tiny->new('default_headers' => \%HEADERS, 'cookie_jar' => $jar);
+ my $response = $request->post($uri, {'content' => $body});
+ _check_return($response->{'status'}, $response->{'content'}, $uri);
+
+ ### Get filter list id
+ $uri = "$ZS_BASE_URI/urlCategories/lite";
+ $response = $request->get($uri);
+ _check_return($response->{'status'}, $response->{'content'}, $uri);
+
+ my $json = JSON::PP->new->utf8->decode($response->{'content'});
+ my $id;
+ for my $item (@{$json}) {
+ if (exists($item->{'configuredName'})) {
+ if ($item->{'configuredName'} eq $ZS_CATEGORY_NAME) {
+ $id = $item->{'id'};
+ }
+ }
+ }
+
+ ### Push Domains
+ $uri = defined($id) ? "$ZS_BASE_URI/urlCategories/$id" : "$ZS_BASE_URI/urlCategories";
+ my $method = defined($id) ? "put" : "post";
+ my $description = "$ZS_CATEGORY_DESC\n\nLast Updated: " . strftime("%Y-%m-%d %H:%M:%S", localtime);
+ $#domains = $#domains >= $ZS_MAX_DOMAINS ? $ZS_MAX_DOMAINS : $#domains;
+ $body = JSON::PP->new->encode({configuredName => $ZS_CATEGORY_NAME, customCategory => 'true', superCategory => 'SECURITY', urls => \@domains, description => $description});
+ $response = $request->$method($uri, {'content' => $body});
+ _check_return($response->{'status'}, $response->{'content'}, $uri);
+
+ ### Delete authenticadSession
+ $uri = "$ZS_BASE_URI/authenticatedSession";
+ $response = $request->delete($uri);
+ _check_return($response->{'status'}, $response->{'content'}, $uri);
+}
+
+say "Running in $LOGMODE mode..." if $LOGMODE;
+my @domains = netskope();
+zscaler(\@domains);
+mail_csv();
+say "Completed." if $LOGMODE;
blob - /dev/null
blob + 60f318df8ee7ac0d9e8589795023222b3ef16ef1 (mode 755)
--- /dev/null
+++ Netskope_ZScalerImporter-07.pl
+#!/usr/bin/env perl
+#
+# Copyright 2020, Mischa Peters <mischa AT netskope DOT com>, Netskope.
+# Netskope_ZScalerImporter.pl - Version 3.0 - 20200615
+#
+# Permission to use, copy, modify, and distribute this software for any
+# purpose with or without fee is hereby granted, provided that the above
+# copyright notice and this permission notice appear in all copies.
+#
+# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+#
+# ZScaler integration with Netskope
+#
+use 5.024;
+use strict;
+use warnings;
+use autodie;
+use Config::Tiny;
+use Time::HiRes qw(gettimeofday);
+use POSIX qw(strftime);
+use HTTP::Tiny;
+use HTTP::CookieJar;
+use JSON::PP;
+use Text::CSV;
+use MIME::Lite;
+
+my $LOGMODE = "";
+my @CONFIG_FILES = grep { -e } ('./netskope.cnf', './.netskope.cnf', '/etc/netskope.cnf', "$ENV{'HOME'}/.netskope.cnf", "$ENV{'HOME'}/netskope.cnf");
+my $config = Config::Tiny->read($CONFIG_FILES[-1], 'utf8');
+my $USER_COUNT = $config->{report}{USER_COUNT};
+my $MAX_DOMAIN= $config->{report}{MAX_DOMAIN};
+my $NTSKP_TENANT = $config->{netskope}{NTSKP_TENANT};
+my $NTSKP_TOKEN = $config->{netskope}{NTSKP_TOKEN};
+my $NTSKP_REPORTID = $config->{netskope}{NTSKP_REPORTID};
+my $ZS_MAX_DOMAINS = $config->{zscaler}{ZS_MAX_DOMAINS};
+my $ZS_BASE_URI = $config->{zscaler}{ZS_BASE_URI};
+my $ZS_API_KEY = $config->{zscaler}{ZS_API_KEY};
+my $ZS_API_USERNAME = $config->{zscaler}{ZS_API_USERNAME};
+my $ZS_API_PASSWORD = $config->{zscaler}{ZS_API_PASSWORD};
+my $ZS_CATEGORY_NAME = $config->{zscaler}{ZS_CATEGORY_NAME};
+my $ZS_CATEGORY_DESC = $config->{zscaler}{ZS_CATEGORY_DESC};
+my $PROXY = $config->{general}{PROXY};
+my $SMTP = $config->{general}{SMTP};
+my $FROM = $config->{general}{FROM};
+my $TO = $config->{general}{TO};
+my $SUBJECT = $config->{general}{SUBJECT};
+my $TEXT = $config->{general}{TEXT} . "\n\n";
+my %HEADERS = ("Content-Type" => "application/json", "Cache-Control" => "no-cache");
+my $EMAIL_CSV = "";
+
+### Netskope ###
+sub mail_csv {
+ my $msg = MIME::Lite->new(From=> $FROM, To => $TO, Subject => $SUBJECT, Type => 'TEXT', Data => $TEXT);
+ $msg->attach(Type => 'text/csv', Data => $EMAIL_CSV);
+ $msg->send('smtp', $SMTP, Debug=>0);
+ say "MAIL From: $FROM -> $TO - Attach CSV" if $LOGMODE;
+}
+
+sub _check_return {
+ my ($status, $content, $uri) = @_;
+ if ($status =~ /^2/ && $LOGMODE) {
+ print "URI: $uri\nHTTP RESPONSE: $status\n";
+ print "CONTENT:\n$content\n" if ($LOGMODE eq "DEBUG");
+ }
+ if ($status !~ /^2/) {
+ print "URI: $uri\nHTTP RESPONSE: $status\n$content\n";
+ my $msg = MIME::Lite->new(From => $FROM, To => $TO, Subject => 'API Error along the way', Type => 'TEXT',
+ Data => "URI: $uri\nHTTP RESPONSE: $status\n$content\n\n",
+ );
+ $msg->attach(Type => 'text/csv', Data => $EMAIL_CSV);
+ $msg->send('smtp', $SMTP, Debug=>0);
+ say "MAIL From: $FROM -> $TO - Error Notification" if $LOGMODE;
+ say "exit 1";
+ exit 1;
+ }
+}
+
+sub netskope {
+ ### Collect widget IDs
+ my $uri = "$NTSKP_TENANT/api/v1/reports?token=$NTSKP_TOKEN&op=reportInfo&id=$NTSKP_REPORTID";
+ my $request = HTTP::Tiny->new('default_headers' => \%HEADERS);
+ my $response = $request->get($uri);
+ _check_return($response->{'status'}, $response->{'content'}, $uri);
+
+ my $json = JSON::PP->new->utf8->decode($response->{'content'});
+ my $data = $json->{'data'}->{'latestScheduledRunInfo'}->{'widgets'};
+ if (!$data) { _check_return(404, $response->{'content'}, "No Widget Data"); }
+ my %csv_content;
+
+ ### Collect widget data and write to CSV
+ for my $widget (@{$data}) {
+ $uri = "$NTSKP_TENANT/api/v1/reports?token=$NTSKP_TOKEN&op=widgetData&id=$widget->{'id'}";
+ $response = $request->get($uri);
+ print "\n#DEBUG#\nWidget Name: " . $widget->{'name'} . "\nWidgetID: " . $widget->{'id'} . " (ReportID: $NTSKP_REPORTID)\n" . $response->{'content'} if ($LOGMODE && $LOGMODE eq "DEBUG");
+ _check_return($response->{'status'}, $response->{'content'}, $uri);
+ $csv_content{$widget->{'name'}} = $response->{'content'};
+ }
+
+ ### Process domains from CSV
+ my @blocklist;
+ for my $widget_name (keys %csv_content) {
+ my $count = 0;
+ my $csv = Text::CSV->new({binary => 1, auto_diag => 1});
+ open my $fh_in, "<", \$csv_content{$widget_name};
+ $csv->column_names($csv->getline($fh_in));
+ # "Application","Domain","Category","CCI","Users"
+
+ print "\n## Widget Name: $widget_name\n## Domains: " if $LOGMODE;
+ $EMAIL_CSV .= "$widget_name\n";
+ DOMAIN:
+ while (my $row = $csv->getline_hr($fh_in)) {
+ last DOMAIN if ($count == $MAX_DOMAIN);
+ if ($row->{'Users'} < $USER_COUNT) {
+ print "$row->{'Domain'}," if ($LOGMODE ne "DEBUG");
+ print "$row->{'Application'} - $row->{'Domain'} - $row->{'Category'}, $row->{'CCI'}, $row->{'Users'}\n" if ($LOGMODE eq "DEBUG");
+ $EMAIL_CSV .= "$row->{'Domain'},";
+ push @blocklist, $row->{'Domain'};
+ $count++;
+ }
+ }
+ print "\n\n" if $LOGMODE;
+ $EMAIL_CSV .= "\n";
+ }
+ return @blocklist;
+}
+
+### Zscaler ###
+
+sub zscaler {
+ my @domains = @{$_[0]};
+
+ ### Authenticate
+ my $now = int(gettimeofday * 1000);
+ my $n = substr($now, -6);
+ my $r = sprintf "%06d", $n >> 1;
+ my $key;
+ for my $i (0..length($n)-1) {
+ $key .= substr($ZS_API_KEY, substr($n, $i, 1), 1);
+ }
+ for my $i (0..length($r)-1) {
+ $key .= substr($ZS_API_KEY, substr($r, $i, 1) + 2, 1);
+ }
+ my $uri = "$ZS_BASE_URI/authenticatedSession";
+ my $body = JSON::PP->new->encode({apiKey => $key, username => $ZS_API_USERNAME, password => $ZS_API_PASSWORD, timestamp => $now});
+ my $jar = HTTP::CookieJar->new;
+ my $request = HTTP::Tiny->new('default_headers' => \%HEADERS, 'cookie_jar' => $jar);
+ my $response = $request->post($uri, {'content' => $body});
+ _check_return($response->{'status'}, $response->{'content'}, $uri);
+
+ ### Get filter list id
+ $uri = "$ZS_BASE_URI/urlCategories/lite";
+ $response = $request->get($uri);
+ _check_return($response->{'status'}, $response->{'content'}, $uri);
+
+ my $json = JSON::PP->new->utf8->decode($response->{'content'});
+ my $id;
+ for my $item (@{$json}) {
+ if (exists($item->{'configuredName'})) {
+ if ($item->{'configuredName'} eq $ZS_CATEGORY_NAME) {
+ $id = $item->{'id'};
+ }
+ }
+ }
+
+ ### Push Domains
+ $uri = defined($id) ? "$ZS_BASE_URI/urlCategories/$id" : "$ZS_BASE_URI/urlCategories";
+ my $method = defined($id) ? "put" : "post";
+ my $description = "$ZS_CATEGORY_DESC\n\nLast Updated: " . strftime("%Y-%m-%d %H:%M:%S", localtime);
+ splice @domains, $ZS_MAX_DOMAINS if @domains > $ZS_MAX_DOMAINS;
+ $body = JSON::PP->new->encode({configuredName => $ZS_CATEGORY_NAME, customCategory => 'true', superCategory => 'SECURITY', urls => \@domains, description => $description});
+ $response = $request->$method($uri, {'content' => $body});
+ _check_return($response->{'status'}, $response->{'content'}, $uri);
+
+ ### Delete authenticadSession
+ $uri = "$ZS_BASE_URI/authenticatedSession";
+ $response = $request->delete($uri);
+ _check_return($response->{'status'}, $response->{'content'}, $uri);
+}
+
+say "Running in $LOGMODE mode..." if $LOGMODE;
+my @domains = netskope();
+zscaler(\@domains);
+mail_csv();
+say "Completed." if $LOGMODE;
blob - /dev/null
blob + 8b8531e2e44bc856b53d8ebe02253423d1deb173 (mode 755)
--- /dev/null
+++ Netskope_ZScalerImporter-08.pl
+#!/usr/bin/env perl
+#
+# Copyright 2020, Mischa Peters <mischa AT netskope DOT com>, Netskope.
+# Netskope_ZScalerImporter.pl - Version 3.0 - 20200615
+#
+# Permission to use, copy, modify, and distribute this software for any
+# purpose with or without fee is hereby granted, provided that the above
+# copyright notice and this permission notice appear in all copies.
+#
+# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+#
+# ZScaler integration with Netskope
+#
+use 5.016;
+use strict;
+use warnings;
+use autodie;
+use Config::Tiny;
+use Time::HiRes qw(gettimeofday);
+use POSIX qw(strftime);
+use HTTP::Tiny;
+use HTTP::CookieJar;
+use JSON::PP;
+use Text::CSV;
+use MIME::Lite;
+
+my $LOGMODE = "DEBUG";
+my @CONFIG_FILES = grep { -e } ('./netskope.cnf', './.netskope.cnf', '/etc/netskope.cnf', "$ENV{'HOME'}/.netskope.cnf", "$ENV{'HOME'}/netskope.cnf");
+my $config = Config::Tiny->read($CONFIG_FILES[-1], 'utf8');
+my $USER_COUNT = $config->{report}{USER_COUNT};
+my $MAX_DOMAIN= $config->{report}{MAX_DOMAIN};
+my $NTSKP_TENANT = $config->{netskope}{NTSKP_TENANT};
+my $NTSKP_TOKEN = $config->{netskope}{NTSKP_TOKEN};
+my $NTSKP_REPORTID = $config->{netskope}{NTSKP_REPORTID};
+my $ZS_MAX_DOMAINS = $config->{zscaler}{ZS_MAX_DOMAINS};
+my $ZS_BASE_URI = $config->{zscaler}{ZS_BASE_URI};
+my $ZS_API_KEY = $config->{zscaler}{ZS_API_KEY};
+my $ZS_API_USERNAME = $config->{zscaler}{ZS_API_USERNAME};
+my $ZS_API_PASSWORD = $config->{zscaler}{ZS_API_PASSWORD};
+my $ZS_CATEGORY_NAME = $config->{zscaler}{ZS_CATEGORY_NAME};
+my $ZS_CATEGORY_DESC = $config->{zscaler}{ZS_CATEGORY_DESC};
+my $PROXY = $config->{general}{PROXY};
+my $SMTP = $config->{general}{SMTP};
+my $FROM = $config->{general}{FROM};
+my $TO = $config->{general}{TO};
+my $SUBJECT = $config->{general}{SUBJECT};
+my $TEXT = $config->{general}{TEXT} . "\n\n";
+my %HEADERS = ("Content-Type" => "application/json", "Cache-Control" => "no-cache");
+my $EMAIL_CSV = "";
+
+### Netskope ###
+sub mail_csv {
+ my $msg = MIME::Lite->new(From=> $FROM, To => $TO, Subject => $SUBJECT, Type => 'TEXT', Data => $TEXT);
+ $msg->attach(Type => 'text/csv', Data => $EMAIL_CSV, Filename => 'zscaler_blocklist.csv');
+ $msg->send('smtp', $SMTP, Debug=>0);
+ say "MAIL From: $FROM -> $TO - Attach CSV" if $LOGMODE;
+}
+
+sub _check_return {
+ my ($status, $content, $uri) = @_;
+ if ($status =~ /^2/ && $LOGMODE) {
+ print "URI: $uri\nHTTP RESPONSE: $status\n";
+ print "CONTENT:\n$content\n" if ($LOGMODE eq "DEBUG");
+ }
+ if ($status !~ /^2/) {
+ print "URI: $uri\nHTTP RESPONSE: $status\n$content\n";
+ my $msg = MIME::Lite->new(From => $FROM, To => $TO, Subject => 'API Error along the way', Type => 'TEXT',
+ Data => "URI: $uri\nHTTP RESPONSE: $status\n$content\n\n",
+ );
+ $msg->attach(Type => 'text/csv', Data => $EMAIL_CSV);
+ $msg->send('smtp', $SMTP, Debug=>0);
+ say "MAIL From: $FROM -> $TO - Error Notification" if $LOGMODE;
+ say "exit 1";
+ exit 1;
+ }
+}
+
+sub netskope {
+ ### Collect widget IDs
+ my $uri = "$NTSKP_TENANT/api/v1/reports?token=$NTSKP_TOKEN&op=reportInfo&id=$NTSKP_REPORTID";
+ #my $request = HTTP::Tiny->new('default_headers' => \%HEADERS, 'proxy' => $PROXY);
+ my $request = HTTP::Tiny->new('default_headers' => \%HEADERS);
+ my $response = $request->get($uri);
+ _check_return($response->{'status'}, $response->{'content'}, $uri);
+
+ my $json = JSON::PP->new->utf8->decode($response->{'content'});
+ my $data = $json->{'data'}->{'latestScheduledRunInfo'}->{'widgets'};
+ if (!$data) { _check_return(404, $response->{'content'}, "No Widget Data"); }
+ my %csv_content;
+
+ ### Collect widget data and write to CSV
+ for my $widget (@{$data}) {
+ $uri = "$NTSKP_TENANT/api/v1/reports?token=$NTSKP_TOKEN&op=widgetData&id=$widget->{'id'}";
+ $response = $request->get($uri);
+ print "\n#DEBUG#\nWidget Name: " . $widget->{'name'} . "\nWidgetID: " . $widget->{'id'} . " (ReportID: $NTSKP_REPORTID)\n" . $response->{'content'} if ($LOGMODE && $LOGMODE eq "DEBUG");
+ _check_return($response->{'status'}, $response->{'content'}, $uri);
+ $csv_content{$widget->{'name'}} = $response->{'content'};
+ }
+
+ ### Process domains from CSV
+ my @blocklist;
+ for my $widget_name (keys %csv_content) {
+ my $count = 0;
+ my $csv = Text::CSV->new({binary => 1, auto_diag => 1});
+ open my $fh_in, "<", \$csv_content{$widget_name};
+ $csv->column_names($csv->getline($fh_in));
+ # "Application","Domain","Category","CCI","Users"
+
+ print "\n## Widget Name: $widget_name\n## Domains: " if $LOGMODE;
+ $EMAIL_CSV .= "$widget_name\n";
+ DOMAIN:
+ while (my $row = $csv->getline_hr($fh_in)) {
+ last DOMAIN if ($count == $MAX_DOMAIN);
+ if ($row->{'Users'} < $USER_COUNT) {
+ print "$row->{'Domain'}," if ($LOGMODE ne "DEBUG");
+ print "$row->{'Application'} - $row->{'Domain'} - $row->{'Category'}, $row->{'CCI'}, $row->{'Users'}\n" if ($LOGMODE eq "DEBUG");
+ if ($row->{'Domain'} =~ /,/) {
+ push @blocklist, split (/,/, $row->{'Domain'});
+ } else {
+ push @blocklist, $row->{'Domain'};
+ }
+ $EMAIL_CSV .= "$row->{'Domain'},";
+ $count++;
+ }
+ }
+ print "\n\n" if $LOGMODE;
+ $EMAIL_CSV .= "\n";
+ }
+ return @blocklist;
+}
+
+### Zscaler ###
+
+sub zscaler {
+ my @domains = @{$_[0]};
+
+ ### Authenticate
+ my $now = int(gettimeofday * 1000);
+ my $n = substr($now, -6);
+ my $r = sprintf "%06d", $n >> 1;
+ my $key;
+ for my $i (0..length($n)-1) {
+ $key .= substr($ZS_API_KEY, substr($n, $i, 1), 1);
+ }
+ for my $i (0..length($r)-1) {
+ $key .= substr($ZS_API_KEY, substr($r, $i, 1) + 2, 1);
+ }
+ my $uri = "$ZS_BASE_URI/authenticatedSession";
+ my $body = JSON::PP->new->encode({apiKey => $key, username => $ZS_API_USERNAME, password => $ZS_API_PASSWORD, timestamp => $now});
+ my $jar = HTTP::CookieJar->new;
+ #my $request = HTTP::Tiny->new('default_headers' => \%HEADERS, 'proxy' => $PROXY, 'cookie_jar' => $jar);
+ my $request = HTTP::Tiny->new('default_headers' => \%HEADERS, 'cookie_jar' => $jar);
+ my $response = $request->post($uri, {'content' => $body});
+ _check_return($response->{'status'}, $response->{'content'}, $uri);
+
+ ### Get filter list id
+ $uri = "$ZS_BASE_URI/urlCategories/lite";
+ $response = $request->get($uri);
+ _check_return($response->{'status'}, $response->{'content'}, $uri);
+
+ my $json = JSON::PP->new->utf8->decode($response->{'content'});
+ my $id;
+ for my $item (@{$json}) {
+ if (exists($item->{'configuredName'})) {
+ if ($item->{'configuredName'} eq $ZS_CATEGORY_NAME) {
+ $id = $item->{'id'};
+ }
+ }
+ }
+
+ ### Push Domains
+ $uri = defined($id) ? "$ZS_BASE_URI/urlCategories/$id" : "$ZS_BASE_URI/urlCategories";
+ my $method = defined($id) ? "put" : "post";
+ my $description = "$ZS_CATEGORY_DESC\n\nLast Updated: " . strftime("%Y-%m-%d %H:%M:%S", localtime);
+ splice @domains, $ZS_MAX_DOMAINS if @domains > $ZS_MAX_DOMAINS;
+ $body = JSON::PP->new->encode({configuredName => $ZS_CATEGORY_NAME, customCategory => 'true', superCategory => 'SECURITY', urls => \@domains, description => $description});
+# DEBUG
+ print "$body\n";
+
+ $response = $request->$method($uri, {'content' => $body});
+ _check_return($response->{'status'}, $response->{'content'}, $uri);
+
+ ### Delete authenticadSession
+ $uri = "$ZS_BASE_URI/authenticatedSession";
+ $response = $request->delete($uri);
+ _check_return($response->{'status'}, $response->{'content'}, $uri);
+}
+
+say "Running in $LOGMODE mode..." if $LOGMODE;
+my @domains = netskope();
+zscaler(\@domains);
+mail_csv();
+say "Completed." if $LOGMODE;
blob - /dev/null
blob + 3a318496d605eb31e6fa8788c75c0e7f8662aba0 (mode 755)
--- /dev/null
+++ Netskope_ZScalerImporter-09.pl
+#!/usr/bin/env perl
+#
+# Copyright 2020, Mischa Peters <mischa AT netskope DOT com>, Netskope.
+# Netskope_ZScalerImporter.pl
+# Version 3.0 - 20200615 - rewrite to Perl
+# Version 3.1 - 20200812 - split domains when comma separated in CSV
+#
+# Permission to use, copy, modify, and distribute this software for any
+# purpose with or without fee is hereby granted, provided that the above
+# copyright notice and this permission notice appear in all copies.
+#
+# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+#
+# ZScaler integration with Netskope
+#
+use 5.016;
+use strict;
+use warnings;
+use autodie;
+use Config::Tiny;
+use Time::HiRes qw(gettimeofday);
+use POSIX qw(strftime);
+use HTTP::Tiny;
+use HTTP::CookieJar;
+use JSON::PP;
+use Text::CSV;
+use MIME::Lite;
+
+my $LOGMODE = "DEBUG";
+my @CONFIG_FILES = grep { -e } ('./netskope.cnf', './.netskope.cnf', '/etc/netskope.cnf', "$ENV{'HOME'}/.netskope.cnf", "$ENV{'HOME'}/netskope.cnf");
+my $config = Config::Tiny->read($CONFIG_FILES[-1], 'utf8');
+my $USER_COUNT = $config->{report}{USER_COUNT};
+my $MAX_DOMAIN= $config->{report}{MAX_DOMAIN};
+my $NTSKP_TENANT = $config->{netskope}{NTSKP_TENANT};
+my $NTSKP_TOKEN = $config->{netskope}{NTSKP_TOKEN};
+my $NTSKP_REPORTID = $config->{netskope}{NTSKP_REPORTID};
+my $ZS_MAX_DOMAINS = $config->{zscaler}{ZS_MAX_DOMAINS};
+my $ZS_BASE_URI = $config->{zscaler}{ZS_BASE_URI};
+my $ZS_API_KEY = $config->{zscaler}{ZS_API_KEY};
+my $ZS_API_USERNAME = $config->{zscaler}{ZS_API_USERNAME};
+my $ZS_API_PASSWORD = $config->{zscaler}{ZS_API_PASSWORD};
+my $ZS_CATEGORY_NAME = $config->{zscaler}{ZS_CATEGORY_NAME};
+my $ZS_CATEGORY_DESC = $config->{zscaler}{ZS_CATEGORY_DESC};
+my $PROXY = $config->{general}{PROXY};
+my $SMTP = $config->{general}{SMTP};
+my $FROM = $config->{general}{FROM};
+my $TO = $config->{general}{TO};
+my $SUBJECT = $config->{general}{SUBJECT};
+my $TEXT = $config->{general}{TEXT} . "\n\n";
+my %HEADERS = ("Content-Type" => "application/json", "Cache-Control" => "no-cache");
+my $EMAIL_CSV = "";
+
+### Netskope ###
+sub mail_csv {
+ my $msg = MIME::Lite->new(From=> $FROM, To => $TO, Subject => $SUBJECT, Type => 'TEXT', Data => $TEXT);
+ $msg->attach(Type => 'text/csv', Data => $EMAIL_CSV, Filename => 'zscaler_blocklist.csv');
+ $msg->send('smtp', $SMTP, Debug=>0);
+ say "MAIL From: $FROM -> $TO - Attach CSV" if $LOGMODE;
+}
+
+sub _check_return {
+ my ($status, $content, $uri) = @_;
+ if ($status =~ /^2/ && $LOGMODE) {
+ print "URI: $uri\nHTTP RESPONSE: $status\n";
+ print "CONTENT:\n$content\n" if ($LOGMODE eq "DEBUG");
+ }
+ if ($status !~ /^2/) {
+ print "URI: $uri\nHTTP RESPONSE: $status\n$content\n";
+ my $msg = MIME::Lite->new(From => $FROM, To => $TO, Subject => 'API Error along the way', Type => 'TEXT',
+ Data => "URI: $uri\nHTTP RESPONSE: $status\n$content\n\n",
+ );
+ $msg->attach(Type => 'text/csv', Data => $EMAIL_CSV);
+ $msg->send('smtp', $SMTP, Debug=>0);
+ say "MAIL From: $FROM -> $TO - Error Notification" if $LOGMODE;
+ say "exit 1";
+ exit 1;
+ }
+}
+
+sub netskope {
+ ### Collect widget IDs
+ my $uri = "$NTSKP_TENANT/api/v1/reports?token=$NTSKP_TOKEN&op=reportInfo&id=$NTSKP_REPORTID";
+ my $request = HTTP::Tiny->new('default_headers' => \%HEADERS, 'proxy' => $PROXY);
+ my $response = $request->get($uri);
+ _check_return($response->{'status'}, $response->{'content'}, $uri);
+
+ my $json = JSON::PP->new->utf8->decode($response->{'content'});
+ my $data = $json->{'data'}->{'latestScheduledRunInfo'}->{'widgets'};
+ if (!$data) { _check_return(404, $response->{'content'}, "No Widget Data"); }
+ my %csv_content;
+
+ ### Collect widget data and write to CSV
+ for my $widget (@{$data}) {
+ $uri = "$NTSKP_TENANT/api/v1/reports?token=$NTSKP_TOKEN&op=widgetData&id=$widget->{'id'}";
+ $response = $request->get($uri);
+ print "\n#DEBUG#\nWidget Name: " . $widget->{'name'} . "\nWidgetID: " . $widget->{'id'} . " (ReportID: $NTSKP_REPORTID)\n" . $response->{'content'} if ($LOGMODE && $LOGMODE eq "DEBUG");
+ _check_return($response->{'status'}, $response->{'content'}, $uri);
+ $csv_content{$widget->{'name'}} = $response->{'content'};
+ }
+
+ ### Process domains from CSV
+ my @blocklist;
+ for my $widget_name (keys %csv_content) {
+ my $count = 0;
+ my $csv = Text::CSV->new({binary => 1, auto_diag => 1});
+ open my $fh_in, "<", \$csv_content{$widget_name};
+ $csv->column_names($csv->getline($fh_in));
+ # "Application","Domain","Category","CCI","Users"
+
+ print "\n## Widget Name: $widget_name\n## Domains: " if $LOGMODE;
+ $EMAIL_CSV .= "$widget_name\n";
+ DOMAIN:
+ while (my $row = $csv->getline_hr($fh_in)) {
+ last DOMAIN if ($count == $MAX_DOMAIN);
+ if ($row->{'Users'} < $USER_COUNT) {
+ print "$row->{'Domain'}," if ($LOGMODE ne "DEBUG");
+ print "$row->{'Application'} - $row->{'Domain'} - $row->{'Category'}, $row->{'CCI'}, $row->{'Users'}\n" if ($LOGMODE eq "DEBUG");
+ if ($row->{'Domain'} =~ /,/) {
+ push @blocklist, split (/,/, $row->{'Domain'});
+ } else {
+ push @blocklist, $row->{'Domain'};
+ }
+ $EMAIL_CSV .= "$row->{'Domain'},";
+ $count++;
+ }
+ }
+ print "\n\n" if $LOGMODE;
+ $EMAIL_CSV .= "\n";
+ }
+ return @blocklist;
+}
+
+### Zscaler ###
+
+sub zscaler {
+ my @domains = @{$_[0]};
+
+ ### Authenticate
+ my $now = int(gettimeofday * 1000);
+ my $n = substr($now, -6);
+ my $r = sprintf "%06d", $n >> 1;
+ my $key;
+ for my $i (0..length($n)-1) {
+ $key .= substr($ZS_API_KEY, substr($n, $i, 1), 1);
+ }
+ for my $i (0..length($r)-1) {
+ $key .= substr($ZS_API_KEY, substr($r, $i, 1) + 2, 1);
+ }
+ my $uri = "$ZS_BASE_URI/authenticatedSession";
+ my $body = JSON::PP->new->encode({apiKey => $key, username => $ZS_API_USERNAME, password => $ZS_API_PASSWORD, timestamp => $now});
+ my $jar = HTTP::CookieJar->new;
+ my $request = HTTP::Tiny->new('default_headers' => \%HEADERS, 'proxy' => $PROXY, 'cookie_jar' => $jar);
+ my $response = $request->post($uri, {'content' => $body});
+ _check_return($response->{'status'}, $response->{'content'}, $uri);
+
+ ### Get filter list id
+ $uri = "$ZS_BASE_URI/urlCategories/lite";
+ $response = $request->get($uri);
+ _check_return($response->{'status'}, $response->{'content'}, $uri);
+
+ my $json = JSON::PP->new->utf8->decode($response->{'content'});
+ my $id;
+ for my $item (@{$json}) {
+ if (exists($item->{'configuredName'})) {
+ if ($item->{'configuredName'} eq $ZS_CATEGORY_NAME) {
+ $id = $item->{'id'};
+ }
+ }
+ }
+
+ ### Push Domains
+ $uri = defined($id) ? "$ZS_BASE_URI/urlCategories/$id" : "$ZS_BASE_URI/urlCategories";
+ my $method = defined($id) ? "put" : "post";
+ my $description = "$ZS_CATEGORY_DESC\n\nLast Updated: " . strftime("%Y-%m-%d %H:%M:%S", localtime);
+ splice @domains, $ZS_MAX_DOMAINS if @domains > $ZS_MAX_DOMAINS;
+ $body = JSON::PP->new->encode({configuredName => $ZS_CATEGORY_NAME, customCategory => 'true', superCategory => 'SECURITY', urls => \@domains, description => $description});
+ print "$body\n" if ($LOGMODE eq "DEBUG");
+
+ $response = $request->$method($uri, {'content' => $body});
+ _check_return($response->{'status'}, $response->{'content'}, $uri);
+
+ ### Delete authenticadSession
+ $uri = "$ZS_BASE_URI/authenticatedSession";
+ $response = $request->delete($uri);
+ _check_return($response->{'status'}, $response->{'content'}, $uri);
+}
+
+say "Running in $LOGMODE mode..." if $LOGMODE;
+my @domains = netskope();
+zscaler(\@domains);
+mail_csv();
+say "Completed." if $LOGMODE;
blob - /dev/null
blob + 9370930d81adb149b59f88e71402575f5f7e1329 (mode 755)
--- /dev/null
+++ Netskope_ZScalerImporter-10.pl
+#!/usr/bin/env perl
+#
+# Copyright 2020, Mischa Peters <mischa AT netskope DOT com>, Netskope.
+# Netskope_ZScalerImporter.pl
+# Version 3.0 - 20200615 - rewrite to Perl
+# Version 3.1 - 20200812 - split domains when comma separated in CSV
+# Version 3.2 - 20200826 - added all fields to CSV export
+#
+# Permission to use, copy, modify, and distribute this software for any
+# purpose with or without fee is hereby granted, provided that the above
+# copyright notice and this permission notice appear in all copies.
+#
+# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+#
+# ZScaler integration with Netskope
+#
+use 5.016;
+use strict;
+use warnings;
+use autodie;
+use Config::Tiny;
+use Time::HiRes qw(gettimeofday);
+use POSIX qw(strftime);
+use HTTP::Tiny;
+use HTTP::CookieJar;
+use JSON::PP;
+use Text::CSV;
+use MIME::Lite;
+
+my $LOGMODE = "DEBUG";
+my @CONFIG_FILES = grep { -e } ('./netskope.cnf', './.netskope.cnf', '/etc/netskope.cnf', "$ENV{'HOME'}/.netskope.cnf", "$ENV{'HOME'}/netskope.cnf");
+my $config = Config::Tiny->read($CONFIG_FILES[-1], 'utf8');
+my $USER_COUNT = $config->{report}{USER_COUNT};
+my $MAX_DOMAIN= $config->{report}{MAX_DOMAIN};
+my $NTSKP_TENANT = $config->{netskope}{NTSKP_TENANT};
+my $NTSKP_TOKEN = $config->{netskope}{NTSKP_TOKEN};
+my $NTSKP_REPORTID = $config->{netskope}{NTSKP_REPORTID};
+my $ZS_MAX_DOMAINS = $config->{zscaler}{ZS_MAX_DOMAINS};
+my $ZS_BASE_URI = $config->{zscaler}{ZS_BASE_URI};
+my $ZS_API_KEY = $config->{zscaler}{ZS_API_KEY};
+my $ZS_API_USERNAME = $config->{zscaler}{ZS_API_USERNAME};
+my $ZS_API_PASSWORD = $config->{zscaler}{ZS_API_PASSWORD};
+my $ZS_CATEGORY_NAME = $config->{zscaler}{ZS_CATEGORY_NAME};
+my $ZS_CATEGORY_DESC = $config->{zscaler}{ZS_CATEGORY_DESC};
+my $PROXY = $config->{general}{PROXY};
+my $SMTP = $config->{general}{SMTP};
+my $FROM = $config->{general}{FROM};
+my $TO = $config->{general}{TO};
+my $SUBJECT = $config->{general}{SUBJECT};
+my $TEXT = $config->{general}{TEXT} . "\n\n";
+my %HEADERS = ("Content-Type" => "application/json", "Cache-Control" => "no-cache");
+my $EMAIL_CSV = "";
+
+### Netskope ###
+sub mail_csv {
+ my $msg = MIME::Lite->new(From=> $FROM, To => $TO, Subject => $SUBJECT, Type => 'TEXT', Data => $TEXT);
+ $msg->attach(Type => 'text/csv', Data => $EMAIL_CSV, Filename => 'zscaler_blocklist.csv');
+ $msg->send('smtp', $SMTP, Debug=>0);
+ say "MAIL From: $FROM -> $TO - Attach CSV" if $LOGMODE;
+}
+
+sub _check_return {
+ my ($status, $content, $uri) = @_;
+ if ($status =~ /^2/ && $LOGMODE) {
+ print "URI: $uri\nHTTP RESPONSE: $status\n";
+ print "CONTENT:\n$content\n" if ($LOGMODE eq "DEBUG");
+ }
+ if ($status !~ /^2/) {
+ print "URI: $uri\nHTTP RESPONSE: $status\n$content\n";
+ my $msg = MIME::Lite->new(From => $FROM, To => $TO, Subject => 'API Error along the way', Type => 'TEXT',
+ Data => "URI: $uri\nHTTP RESPONSE: $status\n$content\n\n",
+ );
+ $msg->attach(Type => 'text/csv', Data => $EMAIL_CSV);
+ $msg->send('smtp', $SMTP, Debug=>0);
+ say "MAIL From: $FROM -> $TO - Error Notification" if $LOGMODE;
+ say "exit 1";
+ exit 1;
+ }
+}
+
+sub netskope {
+ ### Collect widget IDs
+ my $uri = "$NTSKP_TENANT/api/v1/reports?token=$NTSKP_TOKEN&op=reportInfo&id=$NTSKP_REPORTID";
+ my $request = HTTP::Tiny->new('default_headers' => \%HEADERS, 'proxy' => $PROXY);
+ my $response = $request->get($uri);
+ _check_return($response->{'status'}, $response->{'content'}, $uri);
+
+ my $json = JSON::PP->new->utf8->decode($response->{'content'});
+ my $data = $json->{'data'}->{'latestScheduledRunInfo'}->{'widgets'};
+ if (!$data) { _check_return(404, $response->{'content'}, "No Widget Data"); }
+ my %csv_content;
+
+ ### Collect widget data and write to CSV
+ for my $widget (@{$data}) {
+ $uri = "$NTSKP_TENANT/api/v1/reports?token=$NTSKP_TOKEN&op=widgetData&id=$widget->{'id'}";
+ $response = $request->get($uri);
+ print "\n#DEBUG#\nWidget Name: " . $widget->{'name'} . "\nWidgetID: " . $widget->{'id'} . " (ReportID: $NTSKP_REPORTID)\n" . $response->{'content'} if ($LOGMODE && $LOGMODE eq "DEBUG");
+ _check_return($response->{'status'}, $response->{'content'}, $uri);
+ $csv_content{$widget->{'name'}} = $response->{'content'};
+ }
+
+ ### Process domains from CSV
+ my @blocklist;
+ for my $widget_name (keys %csv_content) {
+ my $count = 0;
+ my $domain;
+ my $csv = Text::CSV->new({binary => 1, auto_diag => 1});
+ open my $fh_in, "<", \$csv_content{$widget_name};
+ $csv->column_names($csv->getline($fh_in));
+ # "Application","Domain","Category","CCI","Users"
+
+ print "\n## Widget Name: $widget_name\n## Domains: " if $LOGMODE;
+ $EMAIL_CSV .= "$widget_name\n";
+ $EMAIL_CSV .= "Application,Domain,Category,CCI,Users\n";
+
+ DOMAIN:
+ while (my $row = $csv->getline_hr($fh_in)) {
+ last DOMAIN if ($count == $MAX_DOMAIN);
+ if ($row->{'Users'} < $USER_COUNT) {
+ print "$row->{'Domain'}," if ($LOGMODE ne "DEBUG");
+ print "$row->{'Application'} - $row->{'Domain'} - $row->{'Category'}, $row->{'CCI'}, $row->{'Users'}\n" if ($LOGMODE eq "DEBUG");
+ if ($row->{'Domain'} =~ /,/) {
+ push @blocklist, split (/,/, $row->{'Domain'});
+ $domain = $row->{'Domain'} =~ s/,/ /gr;
+ } else {
+ push @blocklist, $row->{'Domain'};
+ $domain = $row->{'Domain'};
+
+ }
+ #$EMAIL_CSV .= "$row->{'Domain'},";
+ #$EMAIL_CSV .= "$row->{'Application'},$row->{'Domain'},$row->{'Category'},$row->{'CCI'},$row->{'Users'}\n";
+ $EMAIL_CSV .= "$row->{'Application'},$domain,$row->{'Category'},$row->{'CCI'},$row->{'Users'}\n";
+ $count++;
+ }
+ }
+ print "\n\n" if $LOGMODE;
+ $EMAIL_CSV .= "\n";
+ }
+ return @blocklist;
+}
+
+### Zscaler ###
+
+sub zscaler {
+ my @domains = @{$_[0]};
+
+ ### Authenticate
+ my $now = int(gettimeofday * 1000);
+ my $n = substr($now, -6);
+ my $r = sprintf "%06d", $n >> 1;
+ my $key;
+ for my $i (0..length($n)-1) {
+ $key .= substr($ZS_API_KEY, substr($n, $i, 1), 1);
+ }
+ for my $i (0..length($r)-1) {
+ $key .= substr($ZS_API_KEY, substr($r, $i, 1) + 2, 1);
+ }
+ my $uri = "$ZS_BASE_URI/authenticatedSession";
+ my $body = JSON::PP->new->encode({apiKey => $key, username => $ZS_API_USERNAME, password => $ZS_API_PASSWORD, timestamp => $now});
+ my $jar = HTTP::CookieJar->new;
+ my $request = HTTP::Tiny->new('default_headers' => \%HEADERS, 'proxy' => $PROXY, 'cookie_jar' => $jar);
+ my $response = $request->post($uri, {'content' => $body});
+ _check_return($response->{'status'}, $response->{'content'}, $uri);
+
+ ### Get filter list id
+ $uri = "$ZS_BASE_URI/urlCategories/lite";
+ $response = $request->get($uri);
+ _check_return($response->{'status'}, $response->{'content'}, $uri);
+
+ my $json = JSON::PP->new->utf8->decode($response->{'content'});
+ my $id;
+ for my $item (@{$json}) {
+ if (exists($item->{'configuredName'})) {
+ if ($item->{'configuredName'} eq $ZS_CATEGORY_NAME) {
+ $id = $item->{'id'};
+ }
+ }
+ }
+
+ ### Push Domains
+ $uri = defined($id) ? "$ZS_BASE_URI/urlCategories/$id" : "$ZS_BASE_URI/urlCategories";
+ my $method = defined($id) ? "put" : "post";
+ my $description = "$ZS_CATEGORY_DESC\n\nLast Updated: " . strftime("%Y-%m-%d %H:%M:%S", localtime);
+ splice @domains, $ZS_MAX_DOMAINS if @domains > $ZS_MAX_DOMAINS;
+ $body = JSON::PP->new->encode({configuredName => $ZS_CATEGORY_NAME, customCategory => 'true', superCategory => 'SECURITY', urls => \@domains, description => $description});
+ print "$body\n" if ($LOGMODE eq "DEBUG");
+
+ $response = $request->$method($uri, {'content' => $body});
+ _check_return($response->{'status'}, $response->{'content'}, $uri);
+
+ ### Delete authenticadSession
+ $uri = "$ZS_BASE_URI/authenticatedSession";
+ $response = $request->delete($uri);
+ _check_return($response->{'status'}, $response->{'content'}, $uri);
+}
+
+say "Running in $LOGMODE mode..." if $LOGMODE;
+my @domains = netskope();
+#zscaler(\@domains);
+mail_csv();
+say "Completed." if $LOGMODE;
blob - /dev/null
blob + a2a7a5d5caae1acc846a192fdf14dfbe435b4b82 (mode 755)
--- /dev/null
+++ Netskope_ZScalerImporter-11.pl
+#!/usr/bin/env perl
+#
+# Copyright 2020, Mischa Peters <mischa AT netskope DOT com>, Netskope.
+# Netskope_ZScalerImporter.pl
+# Version 3.0 - 20200615 - rewrite to Perl
+# Version 3.1 - 20200812 - split domains when comma separated in CSV
+#
+# Permission to use, copy, modify, and distribute this software for any
+# purpose with or without fee is hereby granted, provided that the above
+# copyright notice and this permission notice appear in all copies.
+#
+# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+#
+# ZScaler integration with Netskope
+#
+use 5.016;
+use strict;
+use warnings;
+use autodie;
+use Config::Tiny;
+use Time::HiRes qw(gettimeofday);
+use POSIX qw(strftime);
+use HTTP::Tiny;
+use HTTP::CookieJar;
+use JSON::PP;
+use Text::CSV;
+use MIME::Lite;
+
+my $LOGMODE = "";
+my @CONFIG_FILES = grep { -e } ('./netskope.cnf', './.netskope.cnf', '/etc/netskope.cnf', "$ENV{'HOME'}/.netskope.cnf", "$ENV{'HOME'}/netskope.cnf");
+my $config = Config::Tiny->read($CONFIG_FILES[-1], 'utf8');
+my $USER_COUNT = $config->{report}{USER_COUNT};
+my $MAX_DOMAIN= $config->{report}{MAX_DOMAIN};
+my $NTSKP_TENANT = $config->{netskope}{NTSKP_TENANT};
+my $NTSKP_TOKEN = $config->{netskope}{NTSKP_TOKEN};
+my $NTSKP_REPORTID = $config->{netskope}{NTSKP_REPORTID};
+my $ZS_MAX_DOMAINS = $config->{zscaler}{ZS_MAX_DOMAINS};
+my $ZS_BASE_URI = $config->{zscaler}{ZS_BASE_URI};
+my $ZS_API_KEY = $config->{zscaler}{ZS_API_KEY};
+my $ZS_API_USERNAME = $config->{zscaler}{ZS_API_USERNAME};
+my $ZS_API_PASSWORD = $config->{zscaler}{ZS_API_PASSWORD};
+my $ZS_CATEGORY_NAME = $config->{zscaler}{ZS_CATEGORY_NAME};
+my $ZS_CATEGORY_DESC = $config->{zscaler}{ZS_CATEGORY_DESC};
+my $PROXY = $config->{general}{PROXY};
+my $SMTP = $config->{general}{SMTP};
+my $FROM = $config->{general}{FROM};
+my $TO = $config->{general}{TO};
+my $SUBJECT = $config->{general}{SUBJECT};
+my $TEXT = $config->{general}{TEXT} . "\n\n";
+my %HEADERS = ("Content-Type" => "application/json", "Cache-Control" => "no-cache");
+my $EMAIL_CSV = "";
+
+### Netskope ###
+sub mail_csv {
+ my $msg = MIME::Lite->new(From=> $FROM, To => $TO, Subject => $SUBJECT, Type => 'TEXT', Data => $TEXT);
+ $msg->attach(Type => 'text/csv', Data => $EMAIL_CSV, Filename => 'zscaler_blocklist.csv');
+ $msg->send('smtp', $SMTP, Debug=>0);
+ say "MAIL From: $FROM -> $TO - Attach CSV" if $LOGMODE;
+}
+
+sub _check_return {
+ my ($status, $content, $uri) = @_;
+ if ($status =~ /^2/ && $LOGMODE) {
+ print "URI: $uri\nHTTP RESPONSE: $status\n";
+ print "CONTENT:\n$content\n" if ($LOGMODE eq "DEBUG");
+ }
+ if ($status !~ /^2/) {
+ print "URI: $uri\nHTTP RESPONSE: $status\n$content\n";
+ my $msg = MIME::Lite->new(From => $FROM, To => $TO, Subject => 'API Error along the way', Type => 'TEXT',
+ Data => "URI: $uri\nHTTP RESPONSE: $status\n$content\n\n",
+ );
+ $msg->attach(Type => 'text/csv', Data => $EMAIL_CSV);
+ $msg->send('smtp', $SMTP, Debug=>0);
+ say "MAIL From: $FROM -> $TO - Error Notification" if $LOGMODE;
+ say "exit 1";
+ exit 1;
+ }
+}
+
+sub netskope {
+ ### Collect widget IDs
+ my $uri = "$NTSKP_TENANT/api/v1/reports?token=$NTSKP_TOKEN&op=reportInfo&id=$NTSKP_REPORTID";
+ my $request = HTTP::Tiny->new('default_headers' => \%HEADERS, 'proxy' => $PROXY);
+ my $response = $request->get($uri);
+ _check_return($response->{'status'}, $response->{'content'}, $uri);
+
+ my $json = JSON::PP->new->utf8->decode($response->{'content'});
+ my $data = $json->{'data'}->{'latestScheduledRunInfo'}->{'widgets'};
+ if (!$data) { _check_return(404, $response->{'content'}, "No Widget Data"); }
+ my %csv_content;
+
+ ### Collect widget data and write to CSV
+ for my $widget (@{$data}) {
+ $uri = "$NTSKP_TENANT/api/v1/reports?token=$NTSKP_TOKEN&op=widgetData&id=$widget->{'id'}";
+ $response = $request->get($uri);
+ print "\n#DEBUG#\nWidget Name: " . $widget->{'name'} . "\nWidgetID: " . $widget->{'id'} . " (ReportID: $NTSKP_REPORTID)\n" . $response->{'content'} if ($LOGMODE && $LOGMODE eq "DEBUG");
+ _check_return($response->{'status'}, $response->{'content'}, $uri);
+ $csv_content{$widget->{'name'}} = $response->{'content'};
+ }
+
+ ### Process domains from CSV
+ my @blocklist;
+ for my $widget_name (keys %csv_content) {
+ my $count = 0;
+ my $domain;
+ my $csv = Text::CSV->new({binary => 1, auto_diag => 1});
+ open my $fh_in, "<", \$csv_content{$widget_name};
+ my @headers = $csv->column_names($csv->getline($fh_in));
+ print "*** ", join(" - ", @headers), " ***\n\n" if $LOGMODE;
+
+ print "\n## Widget Name: $widget_name\n## Domains: " if $LOGMODE;
+ $EMAIL_CSV .= "$widget_name\n";
+ $EMAIL_CSV .= "Application,Domain,Category,CCI,Blocked Events,Users\n";
+
+ DOMAIN:
+ while (my $row = $csv->getline_hr($fh_in)) {
+ last DOMAIN if ($count == $MAX_DOMAIN);
+ next DOMAIN if ($row->{'Blocked Events'} > 0);
+ if ($row->{'Users'} < $USER_COUNT) {
+ print "$row->{'Domain'}," if ($LOGMODE ne "DEBUG");
+ print "$row->{'Application'} - $row->{'Domain'} - $row->{'Category'}, $row->{'CCI'}, $row->{'Blocked Events'}, $row->{'Users'}\n" if ($LOGMODE eq "DEBUG");
+ if ($row->{'Domain'} =~ /,/) {
+ push @blocklist, split (/,/, $row->{'Domain'});
+ $domain = $row->{'Domain'} =~ s/,/ /gr;
+ } else {
+ push @blocklist, $row->{'Domain'};
+ $domain = $row->{'Domain'};
+
+ }
+ $EMAIL_CSV .= "$row->{'Application'},$domain,$row->{'Category'},$row->{'CCI'},$row->{'Blocked Events'},$row->{'Users'}\n";
+ $count++;
+ }
+ }
+ print "\n\n" if $LOGMODE;
+ $EMAIL_CSV .= "\n";
+ }
+ return @blocklist;
+}
+
+### Zscaler ###
+
+sub zscaler {
+ my @domains = @{$_[0]};
+
+ ### Authenticate
+ my $now = int(gettimeofday * 1000);
+ my $n = substr($now, -6);
+ my $r = sprintf "%06d", $n >> 1;
+ my $key;
+ for my $i (0..length($n)-1) {
+ $key .= substr($ZS_API_KEY, substr($n, $i, 1), 1);
+ }
+ for my $i (0..length($r)-1) {
+ $key .= substr($ZS_API_KEY, substr($r, $i, 1) + 2, 1);
+ }
+ my $uri = "$ZS_BASE_URI/authenticatedSession";
+ my $body = JSON::PP->new->encode({apiKey => $key, username => $ZS_API_USERNAME, password => $ZS_API_PASSWORD, timestamp => $now});
+ my $jar = HTTP::CookieJar->new;
+ my $request = HTTP::Tiny->new('default_headers' => \%HEADERS, 'proxy' => $PROXY, 'cookie_jar' => $jar);
+ my $response = $request->post($uri, {'content' => $body});
+ _check_return($response->{'status'}, $response->{'content'}, $uri);
+
+ ### Get filter list id
+ $uri = "$ZS_BASE_URI/urlCategories/lite";
+ $response = $request->get($uri);
+ _check_return($response->{'status'}, $response->{'content'}, $uri);
+
+ my $json = JSON::PP->new->utf8->decode($response->{'content'});
+ my $id;
+ for my $item (@{$json}) {
+ if (exists($item->{'configuredName'})) {
+ if ($item->{'configuredName'} eq $ZS_CATEGORY_NAME) {
+ $id = $item->{'id'};
+ }
+ }
+ }
+
+ ### Push Domains
+ $uri = defined($id) ? "$ZS_BASE_URI/urlCategories/$id" : "$ZS_BASE_URI/urlCategories";
+ my $method = defined($id) ? "put" : "post";
+ my $description = "$ZS_CATEGORY_DESC\n\nLast Updated: " . strftime("%Y-%m-%d %H:%M:%S", localtime);
+ splice @domains, $ZS_MAX_DOMAINS if @domains > $ZS_MAX_DOMAINS;
+ $body = JSON::PP->new->encode({configuredName => $ZS_CATEGORY_NAME, customCategory => 'true', superCategory => 'SECURITY', urls => \@domains, description => $description});
+ print "$body\n" if ($LOGMODE eq "DEBUG");
+
+ $response = $request->$method($uri, {'content' => $body});
+ _check_return($response->{'status'}, $response->{'content'}, $uri);
+
+ ### Delete authenticadSession
+ $uri = "$ZS_BASE_URI/authenticatedSession";
+ $response = $request->delete($uri);
+ _check_return($response->{'status'}, $response->{'content'}, $uri);
+}
+
+say "Running in $LOGMODE mode..." if $LOGMODE;
+my @domains = netskope();
+#zscaler(\@domains);
+mail_csv();
+say "Completed." if $LOGMODE;
blob - /dev/null
blob + 6ff74577bc2654af7e80af830c74c7e147b986a9 (mode 755)
--- /dev/null
+++ Netskope_ZScalerImporter-12.pl
+#!/usr/bin/env perl
+#
+# Copyright 2020, Mischa Peters <mischa AT netskope DOT com>, Netskope.
+# Netskope_ZScalerImporter.pl
+# Version 3.0 - 20200615 - rewrite to Perl
+# Version 3.1 - 20200812 - split domains when comma separated in CSV
+# Version 3.2 - 20200909 - de-duplication of Zscaler URL category
+#
+# Permission to use, copy, modify, and distribute this software for any
+# purpose with or without fee is hereby granted, provided that the above
+# copyright notice and this permission notice appear in all copies.
+#
+# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+#
+# ZScaler integration with Netskope
+#
+use 5.016;
+use strict;
+use warnings;
+use autodie;
+use Config::Tiny;
+use Time::HiRes qw(gettimeofday);
+use POSIX qw(strftime);
+use HTTP::Tiny;
+use HTTP::CookieJar;
+use JSON::PP;
+use Text::CSV;
+use MIME::Lite;
+
+my $LOGMODE = "DEBUG";
+my @CONFIG_FILES = grep { -e } ('./netskope.cnf', './.netskope.cnf', '/etc/netskope.cnf', "$ENV{'HOME'}/.netskope.cnf", "$ENV{'HOME'}/netskope.cnf");
+my $config = Config::Tiny->read($CONFIG_FILES[-1], 'utf8');
+my $USER_COUNT = $config->{report}{USER_COUNT};
+my $MAX_DOMAIN= $config->{report}{MAX_DOMAIN};
+my $NTSKP_TENANT = $config->{netskope}{NTSKP_TENANT};
+my $NTSKP_TOKEN = $config->{netskope}{NTSKP_TOKEN};
+my $NTSKP_REPORTID = $config->{netskope}{NTSKP_REPORTID};
+my $ZS_MAX_DOMAINS = $config->{zscaler}{ZS_MAX_DOMAINS};
+my $ZS_BASE_URI = $config->{zscaler}{ZS_BASE_URI};
+my $ZS_API_KEY = $config->{zscaler}{ZS_API_KEY};
+my $ZS_API_USERNAME = $config->{zscaler}{ZS_API_USERNAME};
+my $ZS_API_PASSWORD = $config->{zscaler}{ZS_API_PASSWORD};
+my $ZS_CATEGORY_NAME = $config->{zscaler}{ZS_CATEGORY_NAME};
+my $ZS_CATEGORY_DESC = $config->{zscaler}{ZS_CATEGORY_DESC};
+my $PROXY = $config->{general}{PROXY};
+my $SMTP = $config->{general}{SMTP};
+my $FROM = $config->{general}{FROM};
+my $TO = $config->{general}{TO};
+my $SUBJECT = $config->{general}{SUBJECT};
+my $TEXT = $config->{general}{TEXT} . "\n\n";
+my %HEADERS = ("Content-Type" => "application/json", "Cache-Control" => "no-cache");
+my $EMAIL_CSV = "";
+
+### Netskope ###
+sub mail_csv {
+ my $msg = MIME::Lite->new(From=> $FROM, To => $TO, Subject => $SUBJECT, Type => 'TEXT', Data => $TEXT);
+ $msg->attach(Type => 'text/csv', Data => $EMAIL_CSV, Filename => 'zscaler_blocklist.csv');
+ $msg->send('smtp', $SMTP, Debug=>0);
+ say "MAIL From: $FROM -> $TO - Attach CSV" if $LOGMODE;
+}
+
+sub _check_return {
+ my ($status, $content, $uri) = @_;
+ if ($status =~ /^2/ && $LOGMODE) {
+ print "URI: $uri\nHTTP RESPONSE: $status\n";
+ print "CONTENT:\n$content\n" if ($LOGMODE eq "DEBUG");
+ }
+ if ($status !~ /^2/) {
+ print "URI: $uri\nHTTP RESPONSE: $status\n$content\n";
+ my $msg = MIME::Lite->new(From => $FROM, To => $TO, Subject => 'API Error along the way', Type => 'TEXT',
+ Data => "URI: $uri\nHTTP RESPONSE: $status\n$content\n\n",
+ );
+ $msg->attach(Type => 'text/csv', Data => $EMAIL_CSV);
+ $msg->send('smtp', $SMTP, Debug=>0);
+ say "MAIL From: $FROM -> $TO - Error Notification" if $LOGMODE;
+ say "exit 1";
+ exit 1;
+ }
+}
+
+sub netskope {
+ my @existing_domains = @{$_[0]};
+
+ ### Collect widget IDs
+ my $uri = "$NTSKP_TENANT/api/v1/reports?token=$NTSKP_TOKEN&op=reportInfo&id=$NTSKP_REPORTID";
+ #my $request = HTTP::Tiny->new('default_headers' => \%HEADERS, 'proxy' => $PROXY);
+ my $request = HTTP::Tiny->new('default_headers' => \%HEADERS);
+ my $response = $request->get($uri);
+ _check_return($response->{'status'}, $response->{'content'}, $uri);
+
+ my $json = JSON::PP->new->utf8->decode($response->{'content'});
+ my $data = $json->{'data'}->{'latestScheduledRunInfo'}->{'widgets'};
+ if (!$data) { _check_return(404, $response->{'content'}, "No Widget Data"); }
+ my %csv_content;
+
+ ### Collect widget data and write to CSV
+ for my $widget (@{$data}) {
+ $uri = "$NTSKP_TENANT/api/v1/reports?token=$NTSKP_TOKEN&op=widgetData&id=$widget->{'id'}";
+ $response = $request->get($uri);
+ print "\n#DEBUG#\nWidget Name: " . $widget->{'name'} . "\nWidgetID: " . $widget->{'id'} . " (ReportID: $NTSKP_REPORTID)\n" . $response->{'content'} if ($LOGMODE && $LOGMODE eq "DEBUG");
+ _check_return($response->{'status'}, $response->{'content'}, $uri);
+ $csv_content{$widget->{'name'}} = $response->{'content'};
+ }
+
+ ### Process domains from CSV
+ my @blocklist;
+ for my $widget_name (keys %csv_content) {
+ my $count = 0;
+ my $domain;
+ my $csv = Text::CSV->new({binary => 1, auto_diag => 1});
+ open my $fh_in, "<", \$csv_content{$widget_name};
+ my @headers = $csv->column_names($csv->getline($fh_in));
+ print "*** ", join(" - ", @headers), " ***\n\n" if $LOGMODE;
+
+ print "\n## Widget Name: $widget_name\n## Domains: " if $LOGMODE;
+ $EMAIL_CSV .= "$widget_name\n";
+ $EMAIL_CSV .= "Application,Domain,Category,CCI,Blocked Events,Users\n";
+
+ DOMAIN: while (my $row = $csv->getline_hr($fh_in)) {
+ last DOMAIN if ($count == $MAX_DOMAIN);
+ #next DOMAIN if ($row->{'Blocked Events'} > 0);
+ if ($row->{'Users'} < $USER_COUNT) {
+ print "$row->{'Domain'}," if ($LOGMODE ne "DEBUG");
+ print "$row->{'Application'} - $row->{'Domain'} - $row->{'Category'}, $row->{'CCI'}, $row->{'Blocked Events'}, $row->{'Users'}\n" if ($LOGMODE eq "DEBUG");
+ if ($row->{'Domain'} =~ /,/) {
+ PARSE:
+ for my $item (split (/,/, $row->{'Domain'})) {
+ next PARSE if (grep(/$item/, @existing_domains));
+ push @blocklist, $item;
+ $domain .= $item . " ";
+ }
+ if ($domain) {
+ $EMAIL_CSV .= "$row->{'Application'},$domain,$row->{'Category'},$row->{'CCI'},$row->{'Blocked Events'},$row->{'Users'}\n";
+ $count++;
+ }
+ } else {
+ next DOMAIN if (!grep(/$row->{'Domain'}/, @existing_domains));
+ push @blocklist, $row->{'Domain'};
+ $EMAIL_CSV .= "$row->{'Application'},$row->{'Domain'},$row->{'Category'},$row->{'CCI'},$row->{'Blocked Events'},$row->{'Users'}\n";
+ $count++;
+ }
+ }
+ }
+ print "\n\n" if $LOGMODE;
+ $EMAIL_CSV .= "\n";
+ print "COUNT: $count\n";
+ }
+ return @blocklist;
+}
+
+### Zscaler ###
+
+sub zscaler_get {
+ ### Authenticate
+ my $now = int(gettimeofday * 1000);
+ my $n = substr($now, -6);
+ my $r = sprintf "%06d", $n >> 1;
+ my $key;
+ for my $i (0..length($n)-1) {
+ $key .= substr($ZS_API_KEY, substr($n, $i, 1), 1);
+ }
+ for my $i (0..length($r)-1) {
+ $key .= substr($ZS_API_KEY, substr($r, $i, 1) + 2, 1);
+ }
+ my $uri = "$ZS_BASE_URI/authenticatedSession";
+ my $body = JSON::PP->new->encode({apiKey => $key, username => $ZS_API_USERNAME, password => $ZS_API_PASSWORD, timestamp => $now});
+ my $jar = HTTP::CookieJar->new;
+ #my $request = HTTP::Tiny->new('default_headers' => \%HEADERS, 'proxy' => $PROXY, 'cookie_jar' => $jar);
+ my $request = HTTP::Tiny->new('default_headers' => \%HEADERS, 'cookie_jar' => $jar);
+ my $response = $request->post($uri, {'content' => $body});
+ _check_return($response->{'status'}, $response->{'content'}, $uri);
+
+ ### Get filter list id
+ $uri = "$ZS_BASE_URI/urlCategories/lite";
+ $response = $request->get($uri);
+ _check_return($response->{'status'}, $response->{'content'}, $uri);
+
+ my $json = JSON::PP->new->utf8->decode($response->{'content'});
+ my $id;
+ for my $item (@{$json}) {
+ if (exists($item->{'configuredName'})) {
+ if ($item->{'configuredName'} eq $ZS_CATEGORY_NAME) {
+ $id = $item->{'id'};
+ }
+ }
+ }
+
+ $uri = "$ZS_BASE_URI/urlCategories/$id";
+ my $method = "get";
+ $response = $request->get($uri);
+ _check_return($response->{'status'}, $response->{'content'}, $uri);
+
+ $json = JSON::PP->new->utf8->decode($response->{'content'});
+ my $data = $json->{'urls'};
+ my @convert = ();
+ for my $item (@{$data}) {
+ push @convert, $item;
+ }
+
+ ### Delete authenticadSession
+ $uri = "$ZS_BASE_URI/authenticatedSession";
+ $response = $request->delete($uri);
+ _check_return($response->{'status'}, $response->{'content'}, $uri);
+
+ return @convert;
+}
+
+
+
+sub zscaler_push {
+ my @domains = @{$_[0]};
+
+ ### Authenticate
+ my $now = int(gettimeofday * 1000);
+ my $n = substr($now, -6);
+ my $r = sprintf "%06d", $n >> 1;
+ my $key;
+ for my $i (0..length($n)-1) {
+ $key .= substr($ZS_API_KEY, substr($n, $i, 1), 1);
+ }
+ for my $i (0..length($r)-1) {
+ $key .= substr($ZS_API_KEY, substr($r, $i, 1) + 2, 1);
+ }
+ my $uri = "$ZS_BASE_URI/authenticatedSession";
+ my $body = JSON::PP->new->encode({apiKey => $key, username => $ZS_API_USERNAME, password => $ZS_API_PASSWORD, timestamp => $now});
+ my $jar = HTTP::CookieJar->new;
+ #my $request = HTTP::Tiny->new('default_headers' => \%HEADERS, 'proxy' => $PROXY, 'cookie_jar' => $jar);
+ my $request = HTTP::Tiny->new('default_headers' => \%HEADERS, 'cookie_jar' => $jar);
+ my $response = $request->post($uri, {'content' => $body});
+ _check_return($response->{'status'}, $response->{'content'}, $uri);
+
+ ### Get filter list id
+ $uri = "$ZS_BASE_URI/urlCategories/lite";
+ $response = $request->get($uri);
+ _check_return($response->{'status'}, $response->{'content'}, $uri);
+
+ my $json = JSON::PP->new->utf8->decode($response->{'content'});
+ my $id;
+ for my $item (@{$json}) {
+ if (exists($item->{'configuredName'})) {
+ if ($item->{'configuredName'} eq $ZS_CATEGORY_NAME) {
+ $id = $item->{'id'};
+ }
+ }
+ }
+
+ ### Push Domains
+ $uri = defined($id) ? "$ZS_BASE_URI/urlCategories/$id?action=ADD_TO_LIST" : "$ZS_BASE_URI/urlCategories";
+ my $method = defined($id) ? "put" : "post";
+ my $description = "$ZS_CATEGORY_DESC\n\nLast Updated: " . strftime("%Y-%m-%d %H:%M:%S", localtime);
+ splice @domains, $ZS_MAX_DOMAINS if @domains > $ZS_MAX_DOMAINS;
+ $body = JSON::PP->new->encode({configuredName => $ZS_CATEGORY_NAME, customCategory => 'true', superCategory => 'SECURITY', urls => \@domains, description => $description});
+ print "$body\n" if ($LOGMODE eq "DEBUG");
+
+ $response = $request->$method($uri, {'content' => $body});
+ _check_return($response->{'status'}, $response->{'content'}, $uri);
+
+ ### Delete authenticadSession
+ $uri = "$ZS_BASE_URI/authenticatedSession";
+ $response = $request->delete($uri);
+ _check_return($response->{'status'}, $response->{'content'}, $uri);
+}
+
+say "Running in $LOGMODE mode..." if $LOGMODE;
+my @existing_domains = zscaler_get();
+my @domains = netskope(\@existing_domains);
+print "Total Domains Pushed: " . scalar @domains . "\n" if $LOGMODE;
+zscaler_push(\@domains);
+mail_csv();
+say "Completed." if $LOGMODE;
blob - /dev/null
blob + f758c2c906d2d92f709691e7fe407ca04a505fff (mode 755)
--- /dev/null
+++ Netskope_ZScalerImporter-13.pl
+#!/usr/bin/env perl
+#
+# Copyright 2020, Mischa Peters <mischa AT netskope DOT com>, Netskope.
+# Netskope_ZScalerImporter.pl
+# Version 3.0 - 20200615 - rewrite to Perl
+# Version 3.1 - 20200812 - split domains when comma separated in CSV
+# Version 3.2 - 20200909 - de-duplication of Zscaler URL category
+#
+# Permission to use, copy, modify, and distribute this software for any
+# purpose with or without fee is hereby granted, provided that the above
+# copyright notice and this permission notice appear in all copies.
+#
+# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+#
+# ZScaler integration with Netskope
+#
+use 5.016;
+use strict;
+use warnings;
+use autodie;
+use Config::Tiny;
+use Time::HiRes qw(gettimeofday);
+use POSIX qw(strftime);
+use HTTP::Tiny;
+use HTTP::CookieJar;
+use JSON::PP;
+use Text::CSV;
+use MIME::Lite;
+
+my $LOGMODE = "DEBUG";
+my @CONFIG_FILES = grep { -e } ('./netskope.cnf', './.netskope.cnf', '/etc/netskope.cnf', "$ENV{'HOME'}/.netskope.cnf", "$ENV{'HOME'}/netskope.cnf");
+my $config = Config::Tiny->read($CONFIG_FILES[-1], 'utf8');
+my $USER_COUNT = $config->{report}{USER_COUNT};
+my $MAX_DOMAIN= $config->{report}{MAX_DOMAIN};
+my $NTSKP_TENANT = $config->{netskope}{NTSKP_TENANT};
+my $NTSKP_TOKEN = $config->{netskope}{NTSKP_TOKEN};
+my $NTSKP_REPORTID = $config->{netskope}{NTSKP_REPORTID};
+my $ZS_MAX_DOMAINS = $config->{zscaler}{ZS_MAX_DOMAINS};
+my $ZS_BASE_URI = $config->{zscaler}{ZS_BASE_URI};
+my $ZS_API_KEY = $config->{zscaler}{ZS_API_KEY};
+my $ZS_API_USERNAME = $config->{zscaler}{ZS_API_USERNAME};
+my $ZS_API_PASSWORD = $config->{zscaler}{ZS_API_PASSWORD};
+my $ZS_CATEGORY_NAME = $config->{zscaler}{ZS_CATEGORY_NAME};
+my $ZS_CATEGORY_DESC = $config->{zscaler}{ZS_CATEGORY_DESC};
+my $PROXY = $config->{general}{PROXY};
+my $SMTP = $config->{general}{SMTP};
+my $FROM = $config->{general}{FROM};
+my $TO = $config->{general}{TO};
+my $SUBJECT = $config->{general}{SUBJECT};
+my $TEXT = $config->{general}{TEXT} . "\n\n";
+my %HEADERS = ("Content-Type" => "application/json", "Cache-Control" => "no-cache");
+my $EMAIL_CSV = "";
+
+### Netskope ###
+sub mail_csv {
+ my $msg = MIME::Lite->new(From=> $FROM, To => $TO, Subject => $SUBJECT, Type => 'TEXT', Data => $TEXT);
+ $msg->attach(Type => 'text/csv', Data => $EMAIL_CSV, Filename => 'zscaler_blocklist.csv');
+ $msg->send('smtp', $SMTP, Debug=>0);
+ say "MAIL From: $FROM -> $TO - Attach CSV" if $LOGMODE;
+}
+
+sub _check_return {
+ my ($status, $content, $uri) = @_;
+ if ($status =~ /^2/ && $LOGMODE) {
+ print "URI: $uri\nHTTP RESPONSE: $status\n";
+ print "CONTENT:\n$content\n" if ($LOGMODE eq "DEBUG");
+ }
+ if ($status !~ /^2/) {
+ print "URI: $uri\nHTTP RESPONSE: $status\n$content\n";
+ my $msg = MIME::Lite->new(From => $FROM, To => $TO, Subject => 'API Error along the way', Type => 'TEXT',
+ Data => "URI: $uri\nHTTP RESPONSE: $status\n$content\n\n",
+ );
+ $msg->attach(Type => 'text/csv', Data => $EMAIL_CSV);
+ $msg->send('smtp', $SMTP, Debug=>0);
+ say "MAIL From: $FROM -> $TO - Error Notification" if $LOGMODE;
+ say "exit 1";
+ exit 1;
+ }
+}
+
+sub netskope {
+ my @existing_domains = @{$_[0]};
+
+ ### Collect widget IDs
+ my $uri = "$NTSKP_TENANT/api/v1/reports?token=$NTSKP_TOKEN&op=reportInfo&id=$NTSKP_REPORTID";
+ my $request = HTTP::Tiny->new('default_headers' => \%HEADERS, 'proxy' => $PROXY);
+ #my $request = HTTP::Tiny->new('default_headers' => \%HEADERS);
+ my $response = $request->get($uri);
+ _check_return($response->{'status'}, $response->{'content'}, $uri);
+
+ my $json = JSON::PP->new->utf8->decode($response->{'content'});
+ my $data = $json->{'data'}->{'latestScheduledRunInfo'}->{'widgets'};
+ if (!$data) { _check_return(404, $response->{'content'}, "No Widget Data"); }
+ my %csv_content;
+
+ ### Collect widget data and write to CSV
+ for my $widget (@{$data}) {
+ $uri = "$NTSKP_TENANT/api/v1/reports?token=$NTSKP_TOKEN&op=widgetData&id=$widget->{'id'}";
+ $response = $request->get($uri);
+ print "\n#DEBUG#\nWidget Name: " . $widget->{'name'} . "\nWidgetID: " . $widget->{'id'} . " (ReportID: $NTSKP_REPORTID)\n" . $response->{'content'} if ($LOGMODE && $LOGMODE eq "DEBUG");
+ _check_return($response->{'status'}, $response->{'content'}, $uri);
+ $csv_content{$widget->{'name'}} = $response->{'content'};
+ }
+
+ ### Process domains from CSV
+ my @blocklist;
+ for my $widget_name (keys %csv_content) {
+ my $count = 0;
+ my $domain;
+ my $csv = Text::CSV->new({binary => 1, auto_diag => 1});
+ open my $fh_in, "<", \$csv_content{$widget_name};
+ my @headers = $csv->column_names($csv->getline($fh_in));
+ print "*** ", join(" - ", @headers), " ***\n\n" if $LOGMODE;
+
+ print "\n## Widget Name: $widget_name\n## Domains: " if $LOGMODE;
+ $EMAIL_CSV .= "$widget_name\n";
+ $EMAIL_CSV .= "Application,Domain,Category,CCI,Blocked Events,Users\n";
+
+ DOMAIN: while (my $row = $csv->getline_hr($fh_in)) {
+ last DOMAIN if ($count == $MAX_DOMAIN);
+ #next DOMAIN if ($row->{'Blocked Events'} > 0);
+ if ($row->{'Users'} < $USER_COUNT) {
+ print "$row->{'Domain'}," if ($LOGMODE ne "DEBUG");
+ print "$row->{'Application'} - $row->{'Domain'} - $row->{'Category'}, $row->{'CCI'}, $row->{'Blocked Events'}, $row->{'Users'}\n" if ($LOGMODE eq "DEBUG");
+ next DOMAIN if ($row->{'Domain'} =~ "n/a");
+ if ($row->{'Domain'} =~ /,/) {
+ PARSE:
+ for my $item (split (/,/, $row->{'Domain'})) {
+ next PARSE if (grep(/$item/, @existing_domains));
+ push @blocklist, $item;
+ $domain .= $item . " ";
+ }
+ if ($domain) {
+ $EMAIL_CSV .= "$row->{'Application'},$domain,$row->{'Category'},$row->{'CCI'},$row->{'Blocked Events'},$row->{'Users'}\n";
+ $count++;
+ }
+ } else {
+ next DOMAIN if (!grep(/$row->{'Domain'}/, @existing_domains));
+ push @blocklist, $row->{'Domain'};
+ $EMAIL_CSV .= "$row->{'Application'},$row->{'Domain'},$row->{'Category'},$row->{'CCI'},$row->{'Blocked Events'},$row->{'Users'}\n";
+ $count++;
+ }
+ }
+ }
+ print "\n\n" if $LOGMODE;
+ $EMAIL_CSV .= "\n";
+ print "COUNT: $count\n";
+ }
+ return @blocklist;
+}
+
+### Zscaler ###
+
+sub zscaler_get {
+ ### Authenticate
+ my $now = int(gettimeofday * 1000);
+ my $n = substr($now, -6);
+ my $r = sprintf "%06d", $n >> 1;
+ my $key;
+ for my $i (0..length($n)-1) {
+ $key .= substr($ZS_API_KEY, substr($n, $i, 1), 1);
+ }
+ for my $i (0..length($r)-1) {
+ $key .= substr($ZS_API_KEY, substr($r, $i, 1) + 2, 1);
+ }
+ my $uri = "$ZS_BASE_URI/authenticatedSession";
+ my $body = JSON::PP->new->encode({apiKey => $key, username => $ZS_API_USERNAME, password => $ZS_API_PASSWORD, timestamp => $now});
+ my $jar = HTTP::CookieJar->new;
+ my $request = HTTP::Tiny->new('default_headers' => \%HEADERS, 'proxy' => $PROXY, 'cookie_jar' => $jar);
+ #my $request = HTTP::Tiny->new('default_headers' => \%HEADERS, 'cookie_jar' => $jar);
+ my $response = $request->post($uri, {'content' => $body});
+ _check_return($response->{'status'}, $response->{'content'}, $uri);
+
+ ### Get filter list id
+ $uri = "$ZS_BASE_URI/urlCategories/lite";
+ $response = $request->get($uri);
+ _check_return($response->{'status'}, $response->{'content'}, $uri);
+
+ my $json = JSON::PP->new->utf8->decode($response->{'content'});
+ my $id;
+ for my $item (@{$json}) {
+ if (exists($item->{'configuredName'})) {
+ if ($item->{'configuredName'} eq $ZS_CATEGORY_NAME) {
+ $id = $item->{'id'};
+ }
+ }
+ }
+
+ $uri = "$ZS_BASE_URI/urlCategories/$id";
+ my $method = "get";
+ $response = $request->get($uri);
+ _check_return($response->{'status'}, $response->{'content'}, $uri);
+
+ $json = JSON::PP->new->utf8->decode($response->{'content'});
+ my $data = $json->{'urls'};
+ my @convert = ();
+ for my $item (@{$data}) {
+ push @convert, $item;
+ }
+
+ ### Delete authenticadSession
+ $uri = "$ZS_BASE_URI/authenticatedSession";
+ $response = $request->delete($uri);
+ _check_return($response->{'status'}, $response->{'content'}, $uri);
+
+ return @convert;
+}
+
+
+
+sub zscaler_push {
+ my @domains = @{$_[0]};
+
+ ### Authenticate
+ my $now = int(gettimeofday * 1000);
+ my $n = substr($now, -6);
+ my $r = sprintf "%06d", $n >> 1;
+ my $key;
+ for my $i (0..length($n)-1) {
+ $key .= substr($ZS_API_KEY, substr($n, $i, 1), 1);
+ }
+ for my $i (0..length($r)-1) {
+ $key .= substr($ZS_API_KEY, substr($r, $i, 1) + 2, 1);
+ }
+ my $uri = "$ZS_BASE_URI/authenticatedSession";
+ my $body = JSON::PP->new->encode({apiKey => $key, username => $ZS_API_USERNAME, password => $ZS_API_PASSWORD, timestamp => $now});
+ my $jar = HTTP::CookieJar->new;
+ my $request = HTTP::Tiny->new('default_headers' => \%HEADERS, 'proxy' => $PROXY, 'cookie_jar' => $jar);
+ #my $request = HTTP::Tiny->new('default_headers' => \%HEADERS, 'cookie_jar' => $jar);
+ my $response = $request->post($uri, {'content' => $body});
+ _check_return($response->{'status'}, $response->{'content'}, $uri);
+
+ ### Get filter list id
+ $uri = "$ZS_BASE_URI/urlCategories/lite";
+ $response = $request->get($uri);
+ _check_return($response->{'status'}, $response->{'content'}, $uri);
+
+ my $json = JSON::PP->new->utf8->decode($response->{'content'});
+ my $id;
+ for my $item (@{$json}) {
+ if (exists($item->{'configuredName'})) {
+ if ($item->{'configuredName'} eq $ZS_CATEGORY_NAME) {
+ $id = $item->{'id'};
+ }
+ }
+ }
+
+ ### Push Domains
+ $uri = defined($id) ? "$ZS_BASE_URI/urlCategories/$id?action=ADD_TO_LIST" : "$ZS_BASE_URI/urlCategories";
+ my $method = defined($id) ? "put" : "post";
+ my $description = "$ZS_CATEGORY_DESC\n\nLast Updated: " . strftime("%Y-%m-%d %H:%M:%S", localtime);
+ splice @domains, $ZS_MAX_DOMAINS if @domains > $ZS_MAX_DOMAINS;
+ $body = JSON::PP->new->encode({configuredName => $ZS_CATEGORY_NAME, customCategory => 'true', superCategory => 'SECURITY', urls => \@domains, description => $description});
+ print "$body\n" if ($LOGMODE eq "DEBUG");
+
+ $response = $request->$method($uri, {'content' => $body});
+ _check_return($response->{'status'}, $response->{'content'}, $uri);
+
+ ### Delete authenticadSession
+ $uri = "$ZS_BASE_URI/authenticatedSession";
+ $response = $request->delete($uri);
+ _check_return($response->{'status'}, $response->{'content'}, $uri);
+}
+
+say "Running in $LOGMODE mode..." if $LOGMODE;
+my @existing_domains = zscaler_get();
+my @domains = netskope(\@existing_domains);
+print "Total Domains Pushed: " . scalar @domains . "\n" if $LOGMODE;
+zscaler_push(\@domains);
+mail_csv();
+say "Completed." if $LOGMODE;
blob - /dev/null
blob + 1e83260ddfa84a1567502d95aee4bb537e963e50 (mode 755)
--- /dev/null
+++ Netskope_ZScalerImporter-14.pl
+#!/usr/bin/env perl
+#
+# Copyright 2020, Mischa Peters <mischa AT netskope DOT com>, Netskope.
+# Netskope_ZScalerImporter.pl
+# Version 3.0 - 20200615 - rewrite to Perl
+# Version 3.1 - 20200812 - split domains when comma separated in CSV
+# Version 3.2 - 20200909 - de-duplication of Zscaler URL category
+# Version 3.3 - 20210121 - filter our entries when domain "n/a"
+#
+# Permission to use, copy, modify, and distribute this software for any
+# purpose with or without fee is hereby granted, provided that the above
+# copyright notice and this permission notice appear in all copies.
+#
+# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+#
+# ZScaler integration with Netskope
+#
+use 5.016;
+use strict;
+use warnings;
+use autodie;
+use Config::Tiny;
+use Time::HiRes qw(gettimeofday);
+use POSIX qw(strftime);
+use HTTP::Tiny;
+use HTTP::CookieJar;
+use JSON::PP;
+use Text::CSV;
+use MIME::Lite;
+
+my $LOGMODE = "DEBUG";
+#my $LOGMODE = "";
+my @CONFIG_FILES = grep { -e } ('./netskope.cnf', './.netskope.cnf', '/etc/netskope.cnf', "$ENV{'HOME'}/.netskope.cnf", "$ENV{'HOME'}/netskope.cnf");
+my $config = Config::Tiny->read($CONFIG_FILES[-1], 'utf8');
+my $USER_COUNT = $config->{report}{USER_COUNT};
+my $MAX_DOMAIN= $config->{report}{MAX_DOMAIN};
+my $NTSKP_TENANT = $config->{netskope}{NTSKP_TENANT};
+my $NTSKP_TOKEN = $config->{netskope}{NTSKP_TOKEN};
+my $NTSKP_REPORTID = $config->{netskope}{NTSKP_REPORTID};
+my $ZS_MAX_DOMAINS = $config->{zscaler}{ZS_MAX_DOMAINS};
+my $ZS_BASE_URI = $config->{zscaler}{ZS_BASE_URI};
+my $ZS_API_KEY = $config->{zscaler}{ZS_API_KEY};
+my $ZS_API_USERNAME = $config->{zscaler}{ZS_API_USERNAME};
+my $ZS_API_PASSWORD = $config->{zscaler}{ZS_API_PASSWORD};
+my $ZS_CATEGORY_NAME = $config->{zscaler}{ZS_CATEGORY_NAME};
+my $ZS_CATEGORY_DESC = $config->{zscaler}{ZS_CATEGORY_DESC};
+my $PROXY = $config->{general}{PROXY};
+my $SMTP = $config->{general}{SMTP};
+my $FROM = $config->{general}{FROM};
+my $TO = $config->{general}{TO};
+my $SUBJECT = $config->{general}{SUBJECT};
+my $TEXT = $config->{general}{TEXT} . "\n\n";
+my %HEADERS = ("Content-Type" => "application/json", "Cache-Control" => "no-cache");
+my $EMAIL_CSV = "";
+
+### Netskope ###
+sub mail_csv {
+ my $msg = MIME::Lite->new(From=> $FROM, To => $TO, Subject => $SUBJECT, Type => 'TEXT', Data => $TEXT);
+ $msg->attach(Type => 'text/csv', Data => $EMAIL_CSV, Filename => 'zscaler_blocklist.csv');
+ $msg->send('smtp', $SMTP, Debug=>0);
+ say "MAIL From: $FROM -> $TO - Attach CSV" if $LOGMODE;
+}
+
+sub _check_return {
+ my ($status, $content, $uri) = @_;
+ if ($status =~ /^2/ && $LOGMODE) {
+ print "URI: $uri\nHTTP RESPONSE: $status\n";
+ print "CONTENT:\n$content\n" if ($LOGMODE eq "DEBUG");
+ }
+ if ($status !~ /^2/) {
+ print "URI: $uri\nHTTP RESPONSE: $status\n$content\n";
+ my $msg = MIME::Lite->new(From => $FROM, To => $TO, Subject => 'API Error along the way', Type => 'TEXT',
+ Data => "URI: $uri\nHTTP RESPONSE: $status\n$content\n\n",
+ );
+ $msg->attach(Type => 'text/csv', Data => $EMAIL_CSV);
+ $msg->send('smtp', $SMTP, Debug=>0);
+ say "MAIL From: $FROM -> $TO - Error Notification" if $LOGMODE;
+ say "exit 1";
+ exit 1;
+ }
+}
+
+sub netskope {
+ my @existing_domains = @{$_[0]};
+
+ ### Collect widget IDs
+ my $uri = "$NTSKP_TENANT/api/v1/reports?token=$NTSKP_TOKEN&op=reportInfo&id=$NTSKP_REPORTID";
+ #my $request = HTTP::Tiny->new('default_headers' => \%HEADERS, 'proxy' => $PROXY);
+ my $request = HTTP::Tiny->new('default_headers' => \%HEADERS);
+ my $response = $request->get($uri);
+ _check_return($response->{'status'}, $response->{'content'}, $uri);
+
+ my $json = JSON::PP->new->utf8->decode($response->{'content'});
+ my $data = $json->{'data'}->{'latestScheduledRunInfo'}->{'widgets'};
+ if (!$data) { _check_return(404, $response->{'content'}, "No Widget Data"); }
+ my %csv_content;
+
+ ### Collect widget data and write to CSV
+ for my $widget (@{$data}) {
+ $uri = "$NTSKP_TENANT/api/v1/reports?token=$NTSKP_TOKEN&op=widgetData&id=$widget->{'id'}";
+ $response = $request->get($uri);
+ print "\n#DEBUG#\nWidget Name: " . $widget->{'name'} . "\nWidgetID: " . $widget->{'id'} . " (ReportID: $NTSKP_REPORTID)\n" . $response->{'content'} if ($LOGMODE && $LOGMODE eq "DEBUG");
+ _check_return($response->{'status'}, $response->{'content'}, $uri);
+ $csv_content{$widget->{'name'}} = $response->{'content'};
+ }
+
+ ### Process domains from CSV
+ my @blocklist;
+ for my $widget_name (keys %csv_content) {
+ my $count = 0;
+ my $domain;
+ my $csv = Text::CSV->new({binary => 1, auto_diag => 1});
+ open my $fh_in, "<", \$csv_content{$widget_name};
+ my @headers = $csv->column_names($csv->getline($fh_in));
+ print "*** ", join(" - ", @headers), " ***\n\n" if $LOGMODE;
+
+ print "\n## Widget Name: $widget_name\n## Domains: " if $LOGMODE;
+ $EMAIL_CSV .= "$widget_name\n";
+ $EMAIL_CSV .= "Application,Domain,Category,CCI,Blocked Events,Users\n";
+
+ DOMAIN: while (my $row = $csv->getline_hr($fh_in)) {
+ last DOMAIN if ($count == $MAX_DOMAIN);
+ next DOMAIN if ($row->{'Blocked Events'} > 0);
+ if ($row->{'Users'} < $USER_COUNT) {
+ print "$row->{'Domain'}," if ($LOGMODE ne "DEBUG");
+ print "$row->{'Application'} - $row->{'Domain'} - $row->{'Category'}, $row->{'CCI'}, $row->{'Blocked Events'}, $row->{'Users'}\n" if ($LOGMODE eq "DEBUG");
+ next DOMAIN if ($row->{'Domain'} =~ "n/a");
+ if ($row->{'Domain'} =~ /,/) {
+ PARSE:
+ for my $item (split (/,/, $row->{'Domain'})) {
+ next PARSE if (grep(/$item/, @existing_domains));
+ push @blocklist, $item;
+ $domain .= $item . " ";
+ }
+ if ($domain) {
+ $EMAIL_CSV .= "$row->{'Application'},$domain,$row->{'Category'},$row->{'CCI'},$row->{'Blocked Events'},$row->{'Users'}\n";
+ $count++;
+ }
+ } else {
+ next DOMAIN if (grep(/$row->{'Domain'}/, @existing_domains));
+ push @blocklist, $row->{'Domain'};
+ $EMAIL_CSV .= "$row->{'Application'},$row->{'Domain'},$row->{'Category'},$row->{'CCI'},$row->{'Blocked Events'},$row->{'Users'}\n";
+ $count++;
+ }
+ }
+ }
+ print "\n\n" if $LOGMODE;
+ $EMAIL_CSV .= "\n";
+ print "COUNT: $count\n";
+ }
+ return @blocklist;
+}
+
+### Zscaler ###
+
+sub zscaler_get {
+ ### Authenticate
+ my $now = int(gettimeofday * 1000);
+ my $n = substr($now, -6);
+ my $r = sprintf "%06d", $n >> 1;
+ my $key;
+ for my $i (0..length($n)-1) {
+ $key .= substr($ZS_API_KEY, substr($n, $i, 1), 1);
+ }
+ for my $i (0..length($r)-1) {
+ $key .= substr($ZS_API_KEY, substr($r, $i, 1) + 2, 1);
+ }
+ my $uri = "$ZS_BASE_URI/authenticatedSession";
+ my $body = JSON::PP->new->encode({apiKey => $key, username => $ZS_API_USERNAME, password => $ZS_API_PASSWORD, timestamp => $now});
+ my $jar = HTTP::CookieJar->new;
+ #my $request = HTTP::Tiny->new('default_headers' => \%HEADERS, 'proxy' => $PROXY, 'cookie_jar' => $jar);
+ my $request = HTTP::Tiny->new('default_headers' => \%HEADERS, 'cookie_jar' => $jar);
+ my $response = $request->post($uri, {'content' => $body});
+ _check_return($response->{'status'}, $response->{'content'}, $uri);
+
+ ### Get filter list id
+ $uri = "$ZS_BASE_URI/urlCategories/lite";
+ $response = $request->get($uri);
+ _check_return($response->{'status'}, $response->{'content'}, $uri);
+
+ my $json = JSON::PP->new->utf8->decode($response->{'content'});
+ my $id;
+ for my $item (@{$json}) {
+ if (exists($item->{'configuredName'})) {
+ if ($item->{'configuredName'} eq $ZS_CATEGORY_NAME) {
+ $id = $item->{'id'};
+ }
+ }
+ }
+
+ $uri = "$ZS_BASE_URI/urlCategories/$id";
+ my $method = "get";
+ $response = $request->get($uri);
+ _check_return($response->{'status'}, $response->{'content'}, $uri);
+
+ $json = JSON::PP->new->utf8->decode($response->{'content'});
+ my $data = $json->{'urls'};
+ my @convert = ();
+ for my $item (@{$data}) {
+ push @convert, $item;
+ }
+
+ ### Delete authenticadSession
+ $uri = "$ZS_BASE_URI/authenticatedSession";
+ $response = $request->delete($uri);
+ _check_return($response->{'status'}, $response->{'content'}, $uri);
+
+ return @convert;
+}
+
+
+
+sub zscaler_push {
+ my @domains = @{$_[0]};
+
+ ### Authenticate
+ my $now = int(gettimeofday * 1000);
+ my $n = substr($now, -6);
+ my $r = sprintf "%06d", $n >> 1;
+ my $key;
+ for my $i (0..length($n)-1) {
+ $key .= substr($ZS_API_KEY, substr($n, $i, 1), 1);
+ }
+ for my $i (0..length($r)-1) {
+ $key .= substr($ZS_API_KEY, substr($r, $i, 1) + 2, 1);
+ }
+ my $uri = "$ZS_BASE_URI/authenticatedSession";
+ my $body = JSON::PP->new->encode({apiKey => $key, username => $ZS_API_USERNAME, password => $ZS_API_PASSWORD, timestamp => $now});
+ my $jar = HTTP::CookieJar->new;
+ #my $request = HTTP::Tiny->new('default_headers' => \%HEADERS, 'proxy' => $PROXY, 'cookie_jar' => $jar);
+ my $request = HTTP::Tiny->new('default_headers' => \%HEADERS, 'cookie_jar' => $jar);
+ #my $request = HTTP::Tiny->new('default_headers' => \%HEADERS, 'cookie_jar' => $jar);
+ my $response = $request->post($uri, {'content' => $body});
+ _check_return($response->{'status'}, $response->{'content'}, $uri);
+
+ ### Get filter list id
+ $uri = "$ZS_BASE_URI/urlCategories/lite";
+ $response = $request->get($uri);
+ _check_return($response->{'status'}, $response->{'content'}, $uri);
+
+ my $json = JSON::PP->new->utf8->decode($response->{'content'});
+ my $id;
+ for my $item (@{$json}) {
+ if (exists($item->{'configuredName'})) {
+ if ($item->{'configuredName'} eq $ZS_CATEGORY_NAME) {
+ $id = $item->{'id'};
+ }
+ }
+ }
+
+ ### Push Domains
+ $uri = defined($id) ? "$ZS_BASE_URI/urlCategories/$id?action=ADD_TO_LIST" : "$ZS_BASE_URI/urlCategories?action=ADD_TO_LIST";
+ my $method = defined($id) ? "put" : "post";
+ my $description = "$ZS_CATEGORY_DESC\n\nLast Updated: " . strftime("%Y-%m-%d %H:%M:%S", localtime);
+ splice @domains, $ZS_MAX_DOMAINS if @domains > $ZS_MAX_DOMAINS;
+ $body = JSON::PP->new->encode({configuredName => $ZS_CATEGORY_NAME, customCategory => 'true', superCategory => 'SECURITY', urls => \@domains, description => $description});
+ print "$body\n" if ($LOGMODE eq "DEBUG");
+
+ $response = $request->$method($uri, {'content' => $body});
+ _check_return($response->{'status'}, $response->{'content'}, $uri);
+
+ ### Delete authenticadSession
+ $uri = "$ZS_BASE_URI/authenticatedSession";
+ $response = $request->delete($uri);
+ _check_return($response->{'status'}, $response->{'content'}, $uri);
+}
+
+say "Running in $LOGMODE mode..." if $LOGMODE;
+my @existing_domains = zscaler_get();
+my @domains = netskope(\@existing_domains);
+print "Total Domains Pushed: " . scalar @domains . "\n" if $LOGMODE;
+zscaler_push(\@domains);
+mail_csv();
+say "Completed." if $LOGMODE;
blob - /dev/null
blob + 2f3b9a8b939255b5a2d79d368a7033ff41321f5b (mode 755)
--- /dev/null
+++ Netskope_ZScalerImporter-wip.pl
+#!/usr/bin/env perl
+#
+# Copyright 2020, Mischa Peters <mischa AT netskope DOT com>, Netskope.
+# Netskope_ZScalerImporter.pl
+# Version 3.0 - 20200615 - rewrite to Perl
+# Version 3.1 - 20200812 - split domains when comma separated in CSV
+# Version 3.2 - 20200909 - de-duplication of Zscaler URL category
+#
+# Permission to use, copy, modify, and distribute this software for any
+# purpose with or without fee is hereby granted, provided that the above
+# copyright notice and this permission notice appear in all copies.
+#
+# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+#
+# ZScaler integration with Netskope
+#
+use 5.016;
+use strict;
+use warnings;
+use autodie;
+use Config::Tiny;
+use Time::HiRes qw(gettimeofday);
+use POSIX qw(strftime);
+use HTTP::Tiny;
+use HTTP::CookieJar;
+use JSON::PP;
+use Text::CSV;
+use MIME::Lite;
+
+my $LOGMODE = "DEBUG";
+my @CONFIG_FILES = grep { -e } ('./netskope.cnf', './.netskope.cnf', '/etc/netskope.cnf', "$ENV{'HOME'}/.netskope.cnf", "$ENV{'HOME'}/netskope.cnf");
+my $config = Config::Tiny->read($CONFIG_FILES[-1], 'utf8');
+my $USER_COUNT = $config->{report}{USER_COUNT};
+my $MAX_DOMAIN= $config->{report}{MAX_DOMAIN};
+my $NTSKP_TENANT = $config->{netskope}{NTSKP_TENANT};
+my $NTSKP_TOKEN = $config->{netskope}{NTSKP_TOKEN};
+my $NTSKP_REPORTID = $config->{netskope}{NTSKP_REPORTID};
+my $ZS_MAX_DOMAINS = $config->{zscaler}{ZS_MAX_DOMAINS};
+my $ZS_BASE_URI = $config->{zscaler}{ZS_BASE_URI};
+my $ZS_API_KEY = $config->{zscaler}{ZS_API_KEY};
+my $ZS_API_USERNAME = $config->{zscaler}{ZS_API_USERNAME};
+my $ZS_API_PASSWORD = $config->{zscaler}{ZS_API_PASSWORD};
+my $ZS_CATEGORY_NAME = $config->{zscaler}{ZS_CATEGORY_NAME};
+my $ZS_CATEGORY_DESC = $config->{zscaler}{ZS_CATEGORY_DESC};
+my $PROXY = $config->{general}{PROXY};
+my $SMTP = $config->{general}{SMTP};
+my $FROM = $config->{general}{FROM};
+my $TO = $config->{general}{TO};
+my $SUBJECT = $config->{general}{SUBJECT};
+my $TEXT = $config->{general}{TEXT} . "\n\n";
+my %HEADERS = ("Content-Type" => "application/json", "Cache-Control" => "no-cache");
+my $EMAIL_CSV = "";
+
+### Netskope ###
+sub mail_csv {
+ my $msg = MIME::Lite->new(From=> $FROM, To => $TO, Subject => $SUBJECT, Type => 'TEXT', Data => $TEXT);
+ $msg->attach(Type => 'text/csv', Data => $EMAIL_CSV, Filename => 'zscaler_blocklist.csv');
+ $msg->send('smtp', $SMTP, Debug=>0);
+ say "MAIL From: $FROM -> $TO - Attach CSV" if $LOGMODE;
+}
+
+sub _check_return {
+ my ($status, $content, $uri) = @_;
+ if ($status =~ /^2/ && $LOGMODE) {
+ print "URI: $uri\nHTTP RESPONSE: $status\n";
+ print "CONTENT:\n$content\n" if ($LOGMODE eq "DEBUG");
+ }
+ if ($status !~ /^2/) {
+ print "URI: $uri\nHTTP RESPONSE: $status\n$content\n";
+ my $msg = MIME::Lite->new(From => $FROM, To => $TO, Subject => 'API Error along the way', Type => 'TEXT',
+ Data => "URI: $uri\nHTTP RESPONSE: $status\n$content\n\n",
+ );
+ $msg->attach(Type => 'text/csv', Data => $EMAIL_CSV);
+ $msg->send('smtp', $SMTP, Debug=>0);
+ say "MAIL From: $FROM -> $TO - Error Notification" if $LOGMODE;
+ say "exit 1";
+ exit 1;
+ }
+}
+
+sub netskope {
+ my @existing_domains = @{$_[0]};
+
+ ### Collect widget IDs
+ my $uri = "$NTSKP_TENANT/api/v1/reports?token=$NTSKP_TOKEN&op=reportInfo&id=$NTSKP_REPORTID";
+ #my $request = HTTP::Tiny->new('default_headers' => \%HEADERS, 'proxy' => $PROXY);
+ my $request = HTTP::Tiny->new('default_headers' => \%HEADERS);
+ my $response = $request->get($uri);
+ _check_return($response->{'status'}, $response->{'content'}, $uri);
+
+ my $json = JSON::PP->new->utf8->decode($response->{'content'});
+ my $data = $json->{'data'}->{'latestScheduledRunInfo'}->{'widgets'};
+ if (!$data) { _check_return(404, $response->{'content'}, "No Widget Data"); }
+ my %csv_content;
+
+ ### Collect widget data and write to CSV
+ for my $widget (@{$data}) {
+ $uri = "$NTSKP_TENANT/api/v1/reports?token=$NTSKP_TOKEN&op=widgetData&id=$widget->{'id'}";
+ $response = $request->get($uri);
+ print "\n#DEBUG#\nWidget Name: " . $widget->{'name'} . "\nWidgetID: " . $widget->{'id'} . " (ReportID: $NTSKP_REPORTID)\n" . $response->{'content'} if ($LOGMODE && $LOGMODE eq "DEBUG");
+ _check_return($response->{'status'}, $response->{'content'}, $uri);
+ $csv_content{$widget->{'name'}} = $response->{'content'};
+ }
+
+ ### Process domains from CSV
+ my @blocklist;
+ for my $widget_name (keys %csv_content) {
+ my $count = 0;
+ my $domain;
+ my $csv = Text::CSV->new({binary => 1, auto_diag => 1});
+ open my $fh_in, "<", \$csv_content{$widget_name};
+ my @headers = $csv->column_names($csv->getline($fh_in));
+ print "*** ", join(" - ", @headers), " ***\n\n" if $LOGMODE;
+
+ print "\n## Widget Name: $widget_name\n## Domains: " if $LOGMODE;
+ $EMAIL_CSV .= "$widget_name\n";
+ $EMAIL_CSV .= "Application,Domain,Category,CCI,Blocked Events,Users\n";
+
+ DOMAIN: while (my $row = $csv->getline_hr($fh_in)) {
+ last DOMAIN if ($count == $MAX_DOMAIN);
+ #next DOMAIN if ($row->{'Blocked Events'} > 0);
+ if ($row->{'Users'} < $USER_COUNT) {
+ print "$row->{'Domain'}," if ($LOGMODE ne "DEBUG");
+ print "$row->{'Application'} - $row->{'Domain'} - $row->{'Category'}, $row->{'CCI'}, $row->{'Blocked Events'}, $row->{'Users'}\n" if ($LOGMODE eq "DEBUG");
+ if ($row->{'Domain'} =~ /,/) {
+ PARSE:
+ for my $item (split (/,/, $row->{'Domain'})) {
+ next PARSE if (grep(/$item/, @existing_domains));
+ push @blocklist, $item;
+ $domain .= $item . " ";
+ }
+ if ($domain) {
+ $EMAIL_CSV .= "$row->{'Application'},$domain,$row->{'Category'},$row->{'CCI'},$row->{'Blocked Events'},$row->{'Users'}\n";
+ $count++;
+ }
+ } else {
+ next DOMAIN if (grep(/$row->{'Domain'}/, @existing_domains));
+ push @blocklist, $row->{'Domain'};
+ $EMAIL_CSV .= "$row->{'Application'},$row->{'Domain'},$row->{'Category'},$row->{'CCI'},$row->{'Blocked Events'},$row->{'Users'}\n";
+ $count++;
+ }
+ }
+ }
+ print "\n\n" if $LOGMODE;
+ $EMAIL_CSV .= "\n";
+ print "COUNT: $count\n";
+ }
+ return @blocklist;
+}
+
+### Zscaler ###
+
+sub zscaler_get {
+ ### Authenticate
+ my $now = int(gettimeofday * 1000);
+ my $n = substr($now, -6);
+ my $r = sprintf "%06d", $n >> 1;
+ my $key;
+ for my $i (0..length($n)-1) {
+ $key .= substr($ZS_API_KEY, substr($n, $i, 1), 1);
+ }
+ for my $i (0..length($r)-1) {
+ $key .= substr($ZS_API_KEY, substr($r, $i, 1) + 2, 1);
+ }
+ my $uri = "$ZS_BASE_URI/authenticatedSession";
+ my $body = JSON::PP->new->encode({apiKey => $key, username => $ZS_API_USERNAME, password => $ZS_API_PASSWORD, timestamp => $now});
+ my $jar = HTTP::CookieJar->new;
+ #my $request = HTTP::Tiny->new('default_headers' => \%HEADERS, 'proxy' => $PROXY, 'cookie_jar' => $jar);
+ my $request = HTTP::Tiny->new('default_headers' => \%HEADERS, 'cookie_jar' => $jar);
+ my $response = $request->post($uri, {'content' => $body});
+ _check_return($response->{'status'}, $response->{'content'}, $uri);
+
+ ### Get filter list id
+ $uri = "$ZS_BASE_URI/urlCategories/lite";
+ $response = $request->get($uri);
+ _check_return($response->{'status'}, $response->{'content'}, $uri);
+
+ my $json = JSON::PP->new->utf8->decode($response->{'content'});
+ my $id;
+ for my $item (@{$json}) {
+ if (exists($item->{'configuredName'})) {
+ if ($item->{'configuredName'} eq $ZS_CATEGORY_NAME) {
+ $id = $item->{'id'};
+ }
+ }
+ }
+
+ $uri = "$ZS_BASE_URI/urlCategories/$id";
+ my $method = "get";
+ $response = $request->get($uri);
+ _check_return($response->{'status'}, $response->{'content'}, $uri);
+
+ $json = JSON::PP->new->utf8->decode($response->{'content'});
+ my $data = $json->{'urls'};
+ my @convert = ();
+ for my $item (@{$data}) {
+ push @convert, $item;
+ }
+
+ ### Delete authenticadSession
+ $uri = "$ZS_BASE_URI/authenticatedSession";
+ $response = $request->delete($uri);
+ _check_return($response->{'status'}, $response->{'content'}, $uri);
+
+ return @convert;
+}
+
+
+
+sub zscaler_push {
+ my @domains = @{$_[0]};
+
+ ### Authenticate
+ my $now = int(gettimeofday * 1000);
+ my $n = substr($now, -6);
+ my $r = sprintf "%06d", $n >> 1;
+ my $key;
+ for my $i (0..length($n)-1) {
+ $key .= substr($ZS_API_KEY, substr($n, $i, 1), 1);
+ }
+ for my $i (0..length($r)-1) {
+ $key .= substr($ZS_API_KEY, substr($r, $i, 1) + 2, 1);
+ }
+ my $uri = "$ZS_BASE_URI/authenticatedSession";
+ my $body = JSON::PP->new->encode({apiKey => $key, username => $ZS_API_USERNAME, password => $ZS_API_PASSWORD, timestamp => $now});
+ my $jar = HTTP::CookieJar->new;
+ #my $request = HTTP::Tiny->new('default_headers' => \%HEADERS, 'proxy' => $PROXY, 'cookie_jar' => $jar);
+ my $request = HTTP::Tiny->new('default_headers' => \%HEADERS, 'cookie_jar' => $jar);
+ my $response = $request->post($uri, {'content' => $body});
+ _check_return($response->{'status'}, $response->{'content'}, $uri);
+
+ ### Get filter list id
+ $uri = "$ZS_BASE_URI/urlCategories/lite";
+ $response = $request->get($uri);
+ _check_return($response->{'status'}, $response->{'content'}, $uri);
+
+ my $json = JSON::PP->new->utf8->decode($response->{'content'});
+ my $id;
+ for my $item (@{$json}) {
+ if (exists($item->{'configuredName'})) {
+ if ($item->{'configuredName'} eq $ZS_CATEGORY_NAME) {
+ $id = $item->{'id'};
+ }
+ }
+ }
+
+ ### Push Domains
+ $uri = defined($id) ? "$ZS_BASE_URI/urlCategories/$id?action=ADD_TO_LIST" : "$ZS_BASE_URI/urlCategories";
+ my $method = defined($id) ? "put" : "post";
+ my $description = "$ZS_CATEGORY_DESC\n\nLast Updated: " . strftime("%Y-%m-%d %H:%M:%S", localtime);
+ splice @domains, $ZS_MAX_DOMAINS if @domains > $ZS_MAX_DOMAINS;
+ $body = JSON::PP->new->encode({configuredName => $ZS_CATEGORY_NAME, customCategory => 'true', superCategory => 'SECURITY', urls => \@domains, description => $description});
+ print "$body\n" if ($LOGMODE eq "DEBUG");
+
+ $response = $request->$method($uri, {'content' => $body});
+ _check_return($response->{'status'}, $response->{'content'}, $uri);
+
+ ### Delete authenticadSession
+ $uri = "$ZS_BASE_URI/authenticatedSession";
+ $response = $request->delete($uri);
+ _check_return($response->{'status'}, $response->{'content'}, $uri);
+}
+
+say "Running in $LOGMODE mode..." if $LOGMODE;
+my @existing_domains = zscaler_get();
+my @domains = netskope(\@existing_domains);
+print "Total Domains Pushed: " . scalar @domains . "\n" if $LOGMODE;
+zscaler_push(\@domains);
+mail_csv();
+say "Completed." if $LOGMODE;
blob - /dev/null
blob + df4f396fea482da01932d04befea9fdd03d3c309 (mode 755)
--- /dev/null
+++ httpstat.py
+#!/usr/bin/env python
+# coding: utf-8
+# References:
+# man curl
+# https://curl.haxx.se/libcurl/c/curl_easy_getinfo.html
+# https://curl.haxx.se/libcurl/c/easy_getinfo_options.html
+# http://blog.kenweiner.com/2014/11/http-request-timings-with-curl.html
+
+from __future__ import print_function
+
+import os
+import json
+import sys
+import logging
+import tempfile
+import subprocess
+
+
+__version__ = '1.2.1'
+
+
+PY3 = sys.version_info >= (3,)
+
+if PY3:
+ xrange = range
+
+
+# Env class is copied from https://github.com/reorx/getenv/blob/master/getenv.py
+class Env(object):
+ prefix = 'HTTPSTAT'
+ _instances = []
+
+ def __init__(self, key):
+ self.key = key.format(prefix=self.prefix)
+ Env._instances.append(self)
+
+ def get(self, default=None):
+ return os.environ.get(self.key, default)
+
+
+ENV_SHOW_BODY = Env('{prefix}_SHOW_BODY')
+ENV_SHOW_IP = Env('{prefix}_SHOW_IP')
+ENV_SHOW_SPEED = Env('{prefix}_SHOW_SPEED')
+ENV_SAVE_BODY = Env('{prefix}_SAVE_BODY')
+ENV_CURL_BIN = Env('{prefix}_CURL_BIN')
+ENV_DEBUG = Env('{prefix}_DEBUG')
+
+
+curl_format = """{
+"time_namelookup": %{time_namelookup},
+"time_connect": %{time_connect},
+"time_appconnect": %{time_appconnect},
+"time_pretransfer": %{time_pretransfer},
+"time_redirect": %{time_redirect},
+"time_starttransfer": %{time_starttransfer},
+"time_total": %{time_total},
+"speed_download": %{speed_download},
+"speed_upload": %{speed_upload},
+"remote_ip": "%{remote_ip}",
+"remote_port": "%{remote_port}",
+"local_ip": "%{local_ip}",
+"local_port": "%{local_port}"
+}"""
+
+https_template = """
+ DNS Lookup TCP Connection TLS Handshake Server Processing Content Transfer
+[ {a0000} | {a0001} | {a0002} | {a0003} | {a0004} ]
+ | | | | |
+ namelookup:{b0000} | | | |
+ connect:{b0001} | | |
+ pretransfer:{b0002} | |
+ starttransfer:{b0003} |
+ total:{b0004}
+"""[1:]
+
+http_template = """
+ DNS Lookup TCP Connection Server Processing Content Transfer
+[ {a0000} | {a0001} | {a0003} | {a0004} ]
+ | | | |
+ namelookup:{b0000} | | |
+ connect:{b0001} | |
+ starttransfer:{b0003} |
+ total:{b0004}
+"""[1:]
+
+
+# Color code is copied from https://github.com/reorx/python-terminal-color/blob/master/color_simple.py
+ISATTY = sys.stdout.isatty()
+
+
+def make_color(code):
+ def color_func(s):
+ if not ISATTY:
+ return s
+ tpl = '\x1b[{}m{}\x1b[0m'
+ return tpl.format(code, s)
+ return color_func
+
+
+red = make_color(31)
+green = make_color(32)
+yellow = make_color(33)
+blue = make_color(34)
+magenta = make_color(35)
+cyan = make_color(36)
+
+bold = make_color(1)
+underline = make_color(4)
+
+grayscale = {(i - 232): make_color('38;5;' + str(i)) for i in xrange(232, 256)}
+
+
+def quit(s, code=0):
+ if s is not None:
+ print(s)
+ sys.exit(code)
+
+
+def print_help():
+ help = """
+Usage: httpstat URL [CURL_OPTIONS]
+ httpstat -h | --help
+ httpstat --version
+
+Arguments:
+ URL url to request, could be with or without `http(s)://` prefix
+
+Options:
+ CURL_OPTIONS any curl supported options, except for -w -D -o -S -s,
+ which are already used internally.
+ -h --help show this screen.
+ --version show version.
+
+Environments:
+ HTTPSTAT_SHOW_BODY Set to `true` to show response body in the output,
+ note that body length is limited to 1023 bytes, will be
+ truncated if exceeds. Default is `false`.
+ HTTPSTAT_SHOW_IP By default httpstat shows remote and local IP/port address.
+ Set to `false` to disable this feature. Default is `true`.
+ HTTPSTAT_SHOW_SPEED Set to `true` to show download and upload speed.
+ Default is `false`.
+ HTTPSTAT_SAVE_BODY By default httpstat stores body in a tmp file,
+ set to `false` to disable this feature. Default is `true`
+ HTTPSTAT_CURL_BIN Indicate the curl bin path to use. Default is `curl`
+ from current shell $PATH.
+ HTTPSTAT_DEBUG Set to `true` to see debugging logs. Default is `false`
+"""[1:-1]
+ print(help)
+
+
+def main():
+ args = sys.argv[1:]
+ if not args:
+ print_help()
+ quit(None, 0)
+
+ # get envs
+ show_body = 'true' in ENV_SHOW_BODY.get('false').lower()
+ show_ip = 'true' in ENV_SHOW_IP.get('true').lower()
+ show_speed = 'true'in ENV_SHOW_SPEED.get('false').lower()
+ save_body = 'true' in ENV_SAVE_BODY.get('true').lower()
+ curl_bin = ENV_CURL_BIN.get('curl')
+ is_debug = 'true' in ENV_DEBUG.get('false').lower()
+
+ # configure logging
+ if is_debug:
+ log_level = logging.DEBUG
+ else:
+ log_level = logging.INFO
+ logging.basicConfig(level=log_level)
+ lg = logging.getLogger('httpstat')
+
+ # log envs
+ lg.debug('Envs:\n%s', '\n'.join(' {}={}'.format(i.key, i.get('')) for i in Env._instances))
+ lg.debug('Flags: %s', dict(
+ show_body=show_body,
+ show_ip=show_ip,
+ show_speed=show_speed,
+ save_body=save_body,
+ curl_bin=curl_bin,
+ is_debug=is_debug,
+ ))
+
+ # get url
+ url = args[0]
+ if url in ['-h', '--help']:
+ print_help()
+ quit(None, 0)
+ elif url == '--version':
+ print('httpstat {}'.format(__version__))
+ quit(None, 0)
+
+ curl_args = args[1:]
+
+ # check curl args
+ exclude_options = [
+ '-w', '--write-out',
+ '-D', '--dump-header',
+ '-o', '--output',
+ '-s', '--silent',
+ ]
+ for i in exclude_options:
+ if i in curl_args:
+ quit(yellow('Error: {} is not allowed in extra curl args'.format(i)), 1)
+
+ # tempfile for output
+ bodyf = tempfile.NamedTemporaryFile(delete=False)
+ bodyf.close()
+
+ headerf = tempfile.NamedTemporaryFile(delete=False)
+ headerf.close()
+
+ # run cmd
+ cmd_env = os.environ.copy()
+ cmd_env.update(
+ LC_ALL='C',
+ )
+ cmd_core = [curl_bin, '-w', curl_format, '-D', headerf.name, '-o', bodyf.name, '-s', '-S']
+ cmd = cmd_core + curl_args + [url]
+ lg.debug('cmd: %s', cmd)
+ p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=cmd_env)
+ out, err = p.communicate()
+ if PY3:
+ out, err = out.decode(), err.decode()
+ lg.debug('out: %s', out)
+
+ # print stderr
+ if p.returncode == 0:
+ if err:
+ print(grayscale[16](err))
+ else:
+ _cmd = list(cmd)
+ _cmd[2] = '<output-format>'
+ _cmd[4] = '<tempfile>'
+ _cmd[6] = '<tempfile>'
+ print('> {}'.format(' '.join(_cmd)))
+ quit(yellow('curl error: {}'.format(err)), p.returncode)
+
+ # parse output
+ try:
+ d = json.loads(out)
+ except ValueError as e:
+ print(yellow('Could not decode json: {}'.format(e)))
+ print('curl result:', p.returncode, grayscale[16](out), grayscale[16](err))
+ quit(None, 1)
+ for k in d:
+ if k.startswith('time_'):
+ d[k] = int(d[k] * 1000)
+
+ # calculate ranges
+ d.update(
+ range_dns=d['time_namelookup'],
+ range_connection=d['time_connect'] - d['time_namelookup'],
+ range_ssl=d['time_pretransfer'] - d['time_connect'],
+ range_server=d['time_starttransfer'] - d['time_pretransfer'],
+ range_transfer=d['time_total'] - d['time_starttransfer'],
+ )
+
+ # ip
+ if show_ip:
+ s = 'Connected to {}:{} from {}:{}'.format(
+ cyan(d['remote_ip']), cyan(d['remote_port']),
+ d['local_ip'], d['local_port'],
+ )
+ print(s)
+ print()
+
+ # print header & body summary
+ with open(headerf.name, 'r') as f:
+ headers = f.read().strip()
+ # remove header file
+ lg.debug('rm header file %s', headerf.name)
+ os.remove(headerf.name)
+
+ for loop, line in enumerate(headers.split('\n')):
+ if loop == 0:
+ p1, p2 = tuple(line.split('/'))
+ print(green(p1) + grayscale[14]('/') + cyan(p2))
+ else:
+ pos = line.find(':')
+ print(grayscale[14](line[:pos + 1]) + cyan(line[pos + 1:]))
+
+ print()
+
+ # body
+ if show_body:
+ body_limit = 1024
+ with open(bodyf.name, 'r') as f:
+ body = f.read().strip()
+ body_len = len(body)
+
+ if body_len > body_limit:
+ print(body[:body_limit] + cyan('...'))
+ print()
+ s = '{} is truncated ({} out of {})'.format(green('Body'), body_limit, body_len)
+ if save_body:
+ s += ', stored in: {}'.format(bodyf.name)
+ print(s)
+ else:
+ print(body)
+ else:
+ if save_body:
+ print('{} stored in: {}'.format(green('Body'), bodyf.name))
+
+ # remove body file
+ if not save_body:
+ lg.debug('rm body file %s', bodyf.name)
+ os.remove(bodyf.name)
+
+ # print stat
+ if url.startswith('https://'):
+ template = https_template
+ else:
+ template = http_template
+
+ # colorize template first line
+ tpl_parts = template.split('\n')
+ tpl_parts[0] = grayscale[16](tpl_parts[0])
+ template = '\n'.join(tpl_parts)
+
+ def fmta(s):
+ return cyan('{:^7}'.format(str(s) + 'ms'))
+
+ def fmtb(s):
+ return cyan('{:<7}'.format(str(s) + 'ms'))
+
+ stat = template.format(
+ # a
+ a0000=fmta(d['range_dns']),
+ a0001=fmta(d['range_connection']),
+ a0002=fmta(d['range_ssl']),
+ a0003=fmta(d['range_server']),
+ a0004=fmta(d['range_transfer']),
+ # b
+ b0000=fmtb(d['time_namelookup']),
+ b0001=fmtb(d['time_connect']),
+ b0002=fmtb(d['time_pretransfer']),
+ b0003=fmtb(d['time_starttransfer']),
+ b0004=fmtb(d['time_total']),
+ )
+ print()
+ print(stat)
+
+ # speed, originally bytes per second
+ if show_speed:
+ print('speed_download: {:.1f} KiB/s, speed_upload: {:.1f} KiB/s'.format(
+ d['speed_download'] / 1024, d['speed_upload'] / 1024))
+
+
+if __name__ == '__main__':
+ main()
blob - /dev/null
blob + 9fb047b4eee7de24ffdde30e76c1ff160e74f924 (mode 755)
--- /dev/null
+++ jsondump.py
+#!/usr/bin/env python3
+
+import json
+import urllib.request
+import argparse
+import collections
+from operator import itemgetter
+
+parser = argparse.ArgumentParser(description="API Call to collect data")
+parser.add_argument("tenant", type=str, help="Tenant Name")
+parser.add_argument("token", type=str, help="Tenat API Token")
+parser.add_argument("-t", "--timeperiod", type=int, default='604800', help="Timeperiod (default: 604800)")
+
+try:
+ args = parser.parse_args()
+ tenant = args.tenant
+ token = args.token
+ timeperiod = args.timeperiod
+
+except argparse.ArgumentError as e:
+ print(str(e))
+
+base_url = "https://{}.goskope.com/api/v1/events?token={}&type=page&timeperiod={}".format(tenant, token, timeperiod)
+
+req = urllib.request.Request(base_url)
+with urllib.request.urlopen(req) as response:
+ content = response.read()
+json_content = json.loads(content)
+print(json.dumps(json_content, indent=4, sort_keys=True))
blob - /dev/null
blob + 62d94764579fa2aa96fc769f1b214c821e9081b2 (mode 755)
--- /dev/null
+++ measure.py
+#!/usr/bin/env python3
+#
+# Copyright 2019, Mischa Peters <mischa AT netskope DOT com>, Netskope.
+# Version 1.0 - 20191028
+#
+# Measure timing for DNS lookup as well as HTTP page load
+#
+# Requires:
+# - Python 3.x
+#
+import argparse
+import socket
+import time
+import ssl
+import urllib.request
+from urllib.parse import urlparse
+
+parser = argparse.ArgumentParser(description="Measure load times", epilog="2019 (c) Netskope")
+parser.add_argument("url", type=str, help="url (eg. https://google.com)")
+
+try:
+ args = parser.parse_args()
+ url = args.url
+
+except argparse.ArgumentError as e:
+ print(str(e))
+
+print (url, "timing:")
+
+urlinfo = urlparse(url)
+request_headers = {'Cache-Control': 'no-cache', 'User-Agent': 'Mozilla/5.0'}
+no_cert_check = ssl.create_default_context()
+no_cert_check.check_hostname=False
+no_cert_check.verify_mode=ssl.CERT_NONE
+
+start = time.time()
+ip = socket.gethostbyname(urlinfo.netloc)
+dns_time = time.time()-start
+print ("DNS Lookup:\t{:.3f} seconds".format(dns_time))
+
+start = time.time()
+req = urllib.request.Request(url, headers=request_headers)
+content = urllib.request.urlopen(req, context=no_cert_check).read()
+load_time = time.time()-start
+print ("Page Load:\t{:.3f} seconds".format(load_time))
+print ("w/o DNS Lookup:\t{:.3f} seconds".format(load_time-dns_time))
blob - /dev/null
blob + b84efc9f1822638486f7cd787dab9f977c81d2e3 (mode 755)
--- /dev/null
+++ ns.pl
+#!/usr/bin/perl -w
+use 5.024;
+use strict;
+use warnings;
+use autodie;
+use HTTP::Tiny;
+use Cpanel::JSON::XS;
+
+my $NTSKP_TENANT = "https://oss.de.goskope.com";
+my $NTSKP_TOKEN = "0cfe04c4237cc33dc7f383af5ddbe2e3";
+my $uri = "$NTSKP_TENANT/api/v1/alerts?token=$NTSKP_TOKEN&timeperiod=86400&groupby=application&query=access_method+eq+Client+and+action+eq+block";
+#my $uri = "$NTSKP_TENANT/api/v1/report?token=$NTSKP_TOKEN&timeperiod=86400&type=connection&groupby=application&query=app-cci-app-tag+eq+'Under_Review'";
+#my $uri = "$NTSKP_TENANT/api/v1/report?token=$NTSKP_TOKEN&timeperiod=86400&type=connection&groupby=application&query=app-cci-app-tag+eq+'Pending_GRC_Review'";
+my $response = HTTP::Tiny->new->get($uri);
+my $json = Cpanel::JSON::XS->new->utf8->decode($response->{'content'});
+my $data = $json->{'data'};
+for my $item (@{$data}) {
+ if (exists($item->{'app'})) {
+ print ".";
+ #print $item->{'app'} . ", ";
+ #say $item->{'sessions'};
+ }
+}
+say "";
blob - /dev/null
blob + f515bbe2cf74e4686ff66d6fbafde3e719a0afa6 (mode 755)
--- /dev/null
+++ ntskp-api-01.pl
+#!/usr/bin/perl -w
+use strict;
+use warnings;
+use autodie;
+use POSIX qw(strftime);
+use Cpanel::JSON::XS;
+
+my $file;
+{
+ local $/;
+ open my $fh, "<", "amsjson.txt";
+ $file = <$fh>;
+ close $fh;
+}
+
+my $json = Cpanel::JSON::XS->new->utf8->decode($file);
+my $data = $json->{'data'};
+my $domain;
+my $cci;
+
+for (my $i = 0; $i < (@{$data}); $i++) {
+ #print "Timestamp: $data->[$i]->{'timestamp'}\n";
+ #print "Domain: $data->[$i]->{'domain'}\n";
+
+ if (!$data->[$i]->{'domain'}) {
+ my $url = $data->[$i]->{'url'};
+ $url =~ s!^https?://(?:www\.)?!!i;
+ $url =~ s!/.*!!;
+ $url =~ s/[\?\#\:].*//;
+ $domain = $url;
+ } else {
+ $domain = $data->[$i]->{'domain'};
+ }
+ if ($data->[$i]->{'cci'}) {
+ $cci = $data->[$i]->{'cci'};
+ } else {
+ $cci = 'none';
+ }
+
+ #print "Category: $data->[$i]->{'category'}\n";
+ #print "CCI: $data->[$i]->{'ccl'}\n";
+ #print "User: $data->[$i]->{'user'}\n";
+ my $timestamp = strftime("%Y-%m-%d %H:%M:%S", gmtime($data->[$i]->{'timestamp'}));
+ print "$timestamp,$domain,$cci,$data->[$i]->{'category'},$data->[$i]->{'ccl'}\n";
+}
blob - /dev/null
blob + 1dce7d97d054fa92e96a72c508db52676b5c4c5e (mode 755)
--- /dev/null
+++ ntskp-api-02.pl
+#!/usr/bin/perl -w
+use strict;
+use warnings;
+use autodie;
+use POSIX qw(strftime);
+use Config::Tiny;
+use HTTP::Tiny;
+use Cpanel::JSON::XS;
+
+my $CONFIG_FILE = "/home/mischa/netskope/netskope.cnf";
+my $config = Config::Tiny->read($CONFIG_FILE, 'utf8');
+my $NTSKP_TENANT = $config->{netskope}{NTSKP_TENANT};
+my $NTSKP_TOKEN = $config->{netskope}{NTSKP_TOKEN};
+my $NTSKP_PERIOD = $config->{netskope}{NTSKP_PERIOD};
+my $NTSKP_SCORE = $config->{netskope}{NTSKP_SCORE};
+my $NTSKP_CATEGORIES = $config->{netskope}{NTSKP_CATEGORIES};
+
+my $uri;
+my $skip = 0;
+my $response;
+my $json;
+my $data;
+my $length;
+my $domain;
+my $cci;
+
+my $file_out = "extracted-" . strftime("%Y%m%d", localtime) . ".txt";
+print "File: $file_out\n";
+print "Tenant: $NTSKP_TENANT\n";
+
+while ($skip < 500000) {
+ $uri = "$NTSKP_TENANT/api/v1/events?token=$NTSKP_TOKEN&type=page&timeperiod=$NTSKP_PERIOD&skip=$skip";
+ $response = HTTP::Tiny->new->get($uri);
+ print "HTTP: $response->{status} $response->{reason}\n";
+ $json = Cpanel::JSON::XS->new->utf8->decode($response->{content});
+ print "API: $json->{'status'}\n";
+ $data = $json->{'data'};
+
+ $length = (@{$data});
+ if ($length == 0) {
+ print "All data collected\n";
+ last;
+ }
+
+ open my $fh_out, ">>", $file_out;
+ for (my $i = 0; $i < $length; $i++) {
+ if (!$data->[$i]->{'domain'}) {
+ my $url = $data->[$i]->{'url'};
+ $url =~ s!^https?://(?:www\.)?!!i;
+ $url =~ s!/.*!!;
+ $url =~ s/[\?\#\:].*//;
+ $domain = $url;
+ } else {
+ $domain = $data->[$i]->{'domain'};
+ }
+ if ($data->[$i]->{'cci'}) {
+ $cci = $data->[$i]->{'cci'};
+ } else {
+ $cci = 'none';
+ }
+
+ my $timestamp = strftime("%Y-%m-%d %H:%M:%S", gmtime($data->[$i]->{'timestamp'}));
+ print $fh_out "$timestamp,$domain,$cci,$data->[$i]->{'category'},$data->[$i]->{'ccl'},$data->[$i]->{'user'}\n";
+ }
+ close $fh_out;
+ $skip += 5000;
+ #print "Next batch $skip\n";
+}
+print "Done\n";
blob - /dev/null
blob + f0bccc6d2e7b493df7ce8475de637a08ba84b2e9 (mode 755)
--- /dev/null
+++ ntskp-api-03.pl
+#!/usr/bin/perl -w
+use 5.024;
+use strict;
+use warnings;
+use autodie;
+use Config::Tiny;
+use HTTP::Tiny;
+#use Cpanel::JSON::XS;
+use JSON::PP;
+
+my $CONFIG_FILE = "/home/mischa/netskope/netskope.cnf";
+my $config = Config::Tiny->read($CONFIG_FILE, 'utf8');
+my $NTSKP_TENANT = $config->{netskope}->{NTSKP_TENANT};
+my $NTSKP_TOKEN = $config->{netskope}->{NTSKP_TOKEN};
+my $NTSKP_PERIOD = $config->{netskope}->{NTSKP_PERIOD};
+
+my $uri = "$NTSKP_TENANT/api/v1/events?token=$NTSKP_TOKEN&type=page&timeperiod=$NTSKP_PERIOD";
+my $response = HTTP::Tiny->new->get($uri);
+#my $json = Cpanel::JSON::XS->new->indent(1)->encode($response->{content});
+my $json = JSON::PP->new->pretty(1)->encode($response->{content});
+print $json;
blob - /dev/null
blob + 181269447a0efaa4599e8591ccb187c95b2e33e1 (mode 755)
--- /dev/null
+++ ntskp-api-04.pl
+#!/usr/bin/perl -w
+use 5.024;
+use strict;
+use warnings;
+use autodie;
+use Config::Tiny;
+use HTTP::Tiny;
+#use Cpanel::JSON::XS;
+use JSON::PP;
+
+my $CONFIG_FILE = "/home/mischa/netskope/netskope.cnf";
+my $config = Config::Tiny->read($CONFIG_FILE, 'utf8');
+my $NTSKP_TENANT = $config->{netskope}->{NTSKP_TENANT};
+my $NTSKP_TOKEN = $config->{netskope}->{NTSKP_TOKEN};
+my $NTSKP_PERIOD = $config->{netskope}->{NTSKP_PERIOD};
+
+my $uri = "$NTSKP_TENANT/api/v1/reports?token=$NTSKP_TOKEN&op=reportInfo&id=498";
+my $response = HTTP::Tiny->new->get($uri);
+#my $json = Cpanel::JSON::XS->new->indent(1)->encode($response->{content});
+my $json = JSON::PP->new->pretty(1)->encode($response->{content});
+print $json;
blob - /dev/null
blob + 5d5d37f3d7dc6e7b00fa32349d0f632352f5ee7a (mode 755)
--- /dev/null
+++ ntskp-api-05.pl
+#!/usr/bin/perl -w
+use strict;
+use warnings;
+use autodie;
+use POSIX qw(strftime);
+use Config::Tiny;
+use HTTP::Tiny;
+use JSON::PP;
+
+my $CONFIG_FILE = "/home/mischa/netskope/netskope.cnf";
+my $config = Config::Tiny->read($CONFIG_FILE, 'utf8');
+my $NTSKP_TENANT = $config->{netskope}{NTSKP_TENANT};
+my $NTSKP_TOKEN = $config->{netskope}{NTSKP_TOKEN};
+my $NTSKP_PERIOD = $config->{netskope}{NTSKP_PERIOD};
+my $NTSKP_SCORE = $config->{netskope}{NTSKP_SCORE};
+my $NTSKP_CATEGORIES = $config->{netskope}{NTSKP_CATEGORIES};
+my $from_email = 'mischa@netskope.com';
+my $to_email = 'mischa@netskope.com';
+my $subject = "AZ Blocklist Report";
+
+#print "Tenant: $NTSKP_TENANT\n";
+
+my $uri = "$NTSKP_TENANT/api/v1/reports?token=$NTSKP_TOKEN&op=reportInfo&id=498";
+my $response = HTTP::Tiny->new->get($uri);
+#print "HTTP: $response->{status} $response->{reason}\n";
+my $json = JSON::PP->new->utf8->decode($response->{content});
+#print "API: $json->{'status'}\n";
+my $data = $json->{'data'}->{'latestScheduledRunInfo'}->{'widgets'};
+
+my $length = (@{$data});
+if ($length == 0) {
+ print "No widgets found\n";
+ last;
+}
+
+open my $fh_email, "|-", "/usr/sbin/sendmail -t";
+printf $fh_email "To: %s\n", $to_email;
+printf $fh_email "From: %s\n", $from_email;
+printf $fh_email "Subject: %s\n\n", $subject;
+
+for (my $i = 0; $i < $length; $i++) {
+ print "$data->[$i]->{'id'} - $data->[$i]->{'name'}\n";
+ print $fh_email "$data->[$i]->{'id'} - $data->[$i]->{'name'}\n";
+ $uri = "$NTSKP_TENANT/api/v1/reports?token=$NTSKP_TOKEN&op=widgetData&id=$data->[$i]->{'id'}";
+ $response = HTTP::Tiny->new->get($uri);
+ #print "HTTP: $response->{status} $response->{reason}\n";
+ my $count = 0;
+ foreach (split(/\r\n/, $response->{content})) {
+ last if ($count == 30);
+ my @fields = split(/,/);
+ next if ($fields[1] =~ '"');
+ print "$fields[1],";
+ print $fh_email "$fields[1],";
+ $count++;
+ }
+ print "\n";
+ print $fh_email "\n";
+}
+close $fh_email;
blob - /dev/null
blob + 6d57d61ddf9870d0429af1cfaaa5fdafef8648d9 (mode 755)
--- /dev/null
+++ ntskp-api-06.pl
+#!/usr/bin/perl -w
+use strict;
+use warnings;
+use autodie;
+use POSIX qw(strftime);
+use File::Temp qw/ tempfile tempdir /;
+use Text::CSV;
+
+my $file = "widget-18465-20200611.txt";
+open my $fh, "<", $file;
+
+my $csv = Text::CSV->new({binary => 1, auto_diag => 1});
+
+my $header = $csv->getline($fh);
+
+while (my $row = $csv->getline($fh)) {
+ print "$row->[1]\n";
+}
+close $fh;
+
blob - /dev/null
blob + 2e116c5cbb38cfa43a38c6bbc86afdd7f4a4a7c0 (mode 644)
--- /dev/null
+++ ntskp-phishing.txt
+<html><head>
+<meta http-equiv="Content-Type" content="text/html; charset=us-ascii">
+</head>
+<body style="word-wrap: break-word; -webkit-nbsp-mode: space; line-break: after-white-space;" class="">
+<meta http-equiv="Content-Type" content="text/html; charset=us-ascii" class="">
+<div style="word-wrap: break-word; -webkit-nbsp-mode: space; line-break: after-white-space;" class="">
+Unfortunately, the service you are using has been targeted by a
+very sophisticated password attack. To protect you, the security
+team recommended that we reset all customer passwords immediately.
+Effective immediately, you will be required to reset your service
+password before you can login again. To reset your password please
+use your regular service password reset link.
+<div class=""><br class=""></div>
+<div class="">
+<a href="https://docs.google.com/forms/d/e/1FAIpQLSeuq5BnjifExxv7hmY5yaC0xBDyAk1IlEbxCEnqhG72brnGmQ/viewform?fbzx=-5005650978102270379" class="">Password Reset</a>
+<br class="">
+<div class=""><br class=""></div></div>
+<div class="">Regards,</div>
+<div class=""><br class=""></div>
+<div class="">Your friendly neighbourhood scammer</div>
+<div class=""><br class=""></div></div>
+</body></html>
blob - /dev/null
blob + 0ca90458018bba59228b880d70414d50dfa9e7de (mode 755)
--- /dev/null
+++ ntskp-send.sh
+# high5.nl
+#cat /home/mischa/ntskp-spam.txt | mail -s "$(echo -e "Important\nFrom: Scammer <scammer@local>\nContent-Type: text/html")" mischa@high5.nl
+#cat /home/mischa/ntskp-phishing.txt | mail -s "$(echo -e "Important\nFrom: Scammer <scammer@local>\nContent-Type: text/html")" mischa@high5.nl
+# ntskp.com
+cat /home/mischa/ntskp-spam.txt | mail -s "$(echo -e "Important\nFrom: Scammer <scammer@local>\nContent-Type: text/html")" herman.akker23@ntskp.com
+cat /home/mischa/ntskp-phishing.txt | mail -s "$(echo -e "Important\nFrom: Scammer <scammer@local>\nContent-Type: text/html")" herman.akker23@ntskp.com
+# Microsoft
+cat /home/mischa/ntskp-spam.txt | mail -s "$(echo -e "Important\nFrom: Scammer <scammer@local>\nContent-Type: text/html")" mischa@M365x857260.onmicrosoft.com
+cat /home/mischa/ntskp-phishing.txt | mail -s "$(echo -e "Important\nFrom: Scammer <scammer@local>\nContent-Type: text/html")" mischa@M365x857260.onmicrosoft.com
+# Died
+#cat /home/mischa/ntskp-spam.txt | mail -s "$(echo -e "Important\nFrom: Scammer <scammer@local>\nContent-Type: text/html")" mischa@M365x857260.onmicrosoft.com
+#cat /home/mischa/ntskp-phishing.txt | mail -s "$(echo -e "Important\nFrom: Scammer <scammer@local>\nContent-Type: text/html")" mischa@M365x857260.onmicrosoft.com
blob - /dev/null
blob + 75ebdcfb1a68e1a26a1dc0d28cdef1341f899939 (mode 644)
--- /dev/null
+++ ntskp-spam.txt
+<html><head>
+<meta http-equiv="Content-Type" content="text/html; charset=us-ascii">
+</head>
+<body style="word-wrap: break-word; -webkit-nbsp-mode: space; line-break: after-white-space;" class="">
+<meta http-equiv="Content-Type" content="text/html; charset=us-ascii" class="">
+<div style="word-wrap: break-word; -webkit-nbsp-mode: space; line-break: after-white-space;" class="">
+Please have a look at this CV at your earliest convenience.
+<div class=""><br class=""></div>
+<div class="">
+<a href="http://jn.gs/I7a" class="">LinkedIn CV</a>
+<br class="">
+<div class=""><br class=""></div></div>
+<div class="">Regards,</div>
+<div class=""><br class=""></div>
+<div class="">Your friendly neighbourhood scammer</div>
+<div class=""><br class=""></div></div>
+</body></html>
blob - /dev/null
blob + ee53a735cb6e22578be0bd6578ad63ea7783832c (mode 755)
--- /dev/null
+++ oss1.py
+#!/usr/bin/env python3
+import os
+import sys
+import json
+import requests
+import configparser
+
+###############################################
+# Look for oss.cnf file in current working directory
+CONFIG_FILE = "./oss.cnf"
+if not os.path.isfile(CONFIG_FILE):
+ logging.error(f"The config file {CONFIG_FILE} doesn't exist")
+ sys.exit(1)
+config = configparser.RawConfigParser()
+config.read(CONFIG_FILE)
+NTSKP_TENANT = config.get('netskope', 'NTSKP_TENANT')
+NTSKP_TOKEN = config.get('netskope', 'NTSKP_TOKEN')
+NTSKP_PERIOD = config.get('netskope', 'NTSKP_PERIOD')
+
+###############################################
+
+ssl_session = requests.Session()
+uri = f"{NTSKP_TENANT}/api/v1/alerts?token={NTSKP_TOKEN}&timeperiod={NTSKP_PERIOD}&groupby=application&query=access_method+eq+Client+and+action+eq+block"
+try:
+ r = ssl_session.get(uri)
+ r.raise_for_status()
+except Exception as e:
+ logging.error(f'Error: {str(e)}')
+ sys.exit(1)
+json = r.json()
+if 'data' in json:
+ for item in json['data']:
+ if 'app' in item:
+ print(f"{item['app']} - {item['category']}", end=', ')
+ print()
blob - /dev/null
blob + 867827591b62cca28970b392efe874043993b8b3 (mode 755)
--- /dev/null
+++ oss2.py
+#!/usr/bin/env python3
+import os
+import sys
+import json
+import requests
+import configparser
+
+###############################################
+# Look for oss.cnf file in current working directory
+CONFIG_FILE = "./oss.cnf"
+if not os.path.isfile(CONFIG_FILE):
+ logging.error(f"The config file {CONFIG_FILE} doesn't exist")
+ sys.exit(1)
+config = configparser.RawConfigParser()
+config.read(CONFIG_FILE)
+NTSKP_TENANT = config.get('netskope', 'NTSKP_TENANT')
+NTSKP_TOKEN = config.get('netskope', 'NTSKP_TOKEN')
+NTSKP_PERIOD = config.get('netskope', 'NTSKP_PERIOD')
+
+###############################################
+
+ssl_session = requests.Session()
+uri = f"{NTSKP_TENANT}/api/v1/report?token={NTSKP_TOKEN}&timeperiod={NTSKP_PERIOD}&type=connection&groupby=application&query=app-cci-app-tag+eq+'Under_Review'"
+try:
+ r = ssl_session.get(uri)
+ r.raise_for_status()
+except Exception as e:
+ logging.error(f'Error: {str(e)}')
+ sys.exit(1)
+json = r.json()
+if 'data' in json:
+ for item in json['data']:
+ print(f"{item['app']}", end=', ')
+ print()
blob - /dev/null
blob + c3d9c6635acc23af1b461a72c2ecd187827058a6 (mode 755)
--- /dev/null
+++ oss3.py
+#!/usr/bin/env python3
+import os
+import sys
+import json
+import requests
+import configparser
+
+###############################################
+# Look for oss.cnf file in current working directory
+CONFIG_FILE = "./oss.cnf"
+if not os.path.isfile(CONFIG_FILE):
+ logging.error(f"The config file {CONFIG_FILE} doesn't exist")
+ sys.exit(1)
+config = configparser.RawConfigParser()
+config.read(CONFIG_FILE)
+NTSKP_TENANT = config.get('netskope', 'NTSKP_TENANT')
+NTSKP_TOKEN = config.get('netskope', 'NTSKP_TOKEN')
+NTSKP_PERIOD = config.get('netskope', 'NTSKP_PERIOD')
+
+###############################################
+
+ssl_session = requests.Session()
+uri = f"{NTSKP_TENANT}/api/v1/report?token={NTSKP_TOKEN}&timeperiod={NTSKP_PERIOD}&type=connection&groupby=application&query=app-cci-app-tag+eq+'Pending_GRC_Review'"
+try:
+ r = ssl_session.get(uri)
+ r.raise_for_status()
+except Exception as e:
+ logging.error(f'Error: {str(e)}')
+ sys.exit(1)
+json = r.json()
+if 'data' in json:
+ for item in json['data']:
+ print(f"{item['app']}", end=', ')
+ print()
blob - /dev/null
blob + 2607156aef92b5b5decbb13a10f0c0b02cd529d6 (mode 755)
--- /dev/null
+++ tbi.pl
+#!/usr/bin/env perl
+#
+# Copyright 2020, Mischa Peters <mischa AT netskope DOT com>, Netskope.
+# Netskope_ZScalerImporter.pl - Version 3.0 - 20200615
+#
+# Permission to use, copy, modify, and distribute this software for any
+# purpose with or without fee is hereby granted, provided that the above
+# copyright notice and this permission notice appear in all copies.
+#
+# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+#
+# ZScaler integration with Netskope
+#
+use 5.024;
+use strict;
+use warnings;
+use autodie;
+use Config::Tiny;
+use Time::HiRes qw(gettimeofday);
+use POSIX qw(strftime);
+use HTTP::Tiny;
+use HTTP::CookieJar;
+use JSON::PP;
+use Text::CSV;
+use MIME::Lite;
+
+my $LOGMODE = "";
+#my @CONFIG_FILES = grep { -e } ('./netskope.cnf', './.netskope.cnf', '/etc/netskope.cnf', "$ENV{'HOME'}/.netskope.cnf", "$ENV{'HOME'}/netskope.cnf");
+my @CONFIG_FILES = grep { -e } ('./tbi.cnf');
+my $config = Config::Tiny->read($CONFIG_FILES[-1], 'utf8');
+my $USER_COUNT = $config->{report}{USER_COUNT};
+my $MAX_DOMAIN= $config->{report}{MAX_DOMAIN};
+my $NTSKP_TENANT = $config->{netskope}{NTSKP_TENANT};
+my $NTSKP_TOKEN = $config->{netskope}{NTSKP_TOKEN};
+my $NTSKP_TIMEPERIOD = $config->{netskope}{NTSKP_TIMEPERIOD};
+my $ZS_MAX_DOMAINS = $config->{zscaler}{ZS_MAX_DOMAINS};
+my $ZS_BASE_URI = $config->{zscaler}{ZS_BASE_URI};
+my $ZS_API_KEY = $config->{zscaler}{ZS_API_KEY};
+my $ZS_API_USERNAME = $config->{zscaler}{ZS_API_USERNAME};
+my $ZS_API_PASSWORD = $config->{zscaler}{ZS_API_PASSWORD};
+my $ZS_CATEGORY_NAME = $config->{zscaler}{ZS_CATEGORY_NAME};
+my $ZS_CATEGORY_DESC = $config->{zscaler}{ZS_CATEGORY_DESC};
+my $PROXY = $config->{general}{PROXY};
+my $SMTP = $config->{general}{SMTP};
+my $FROM = $config->{general}{FROM};
+my $TO = $config->{general}{TO};
+my $SUBJECT = $config->{general}{SUBJECT};
+my $TEXT = $config->{general}{TEXT} . "\n\n";
+my %HEADERS = ("Content-Type" => "application/json", "Cache-Control" => "no-cache");
+my $EMAIL_CSV = "";
+
+### Netskope ###
+sub mail_csv {
+ my $msg = MIME::Lite->new(From=> $FROM, To => $TO, Subject => $SUBJECT, Type => 'TEXT', Data => $TEXT);
+ $msg->send('smtp', $SMTP, Debug=>0);
+ say "MAIL From: $FROM -> $TO - Attach CSV" if $LOGMODE;
+}
+
+sub _check_return {
+ my ($status, $content, $uri) = @_;
+ if ($status =~ /^2/ && $LOGMODE) {
+ print "URI: $uri\nHTTP RESPONSE: $status\n";
+ print "CONTENT:\n$content\n" if ($LOGMODE eq "DEBUG");
+ }
+ if ($status !~ /^2/) {
+ print "URI: $uri\nHTTP RESPONSE: $status\n$content\n";
+ my $msg = MIME::Lite->new(From => $FROM, To => $TO, Subject => 'API Error along the way', Type => 'TEXT',
+ Data => "URI: $uri\nHTTP RESPONSE: $status\n$content\n\n",
+ );
+ $msg->send('smtp', $SMTP, Debug=>0);
+ say "MAIL From: $FROM -> $TO - Error Notification" if $LOGMODE;
+ say "exit 1";
+ exit 1;
+ }
+}
+
+sub netskope {
+ my $uri = "$NTSKP_TENANT/api/v1/alerts?token=$NTSKP_TOKEN&timeperiod=$NTSKP_TIMEPERIOD&type=policy";
+ my $request = HTTP::Tiny->new('default_headers' => \%HEADERS);
+ my $response = $request->get($uri);
+ _check_return($response->{'status'}, $response->{'content'}, $uri);
+
+ my $json = JSON::PP->new->utf8->decode($response->{'content'});
+ my $data = $json->{'data'};
+ printf "%-7s %-45s %-35s %s\n", "Action", "Page", "Policy", "Category";
+ say "#############################################################################################################################";
+
+ my @seen;
+ for my $item (@{$data}) {
+ if (exists($item->{'page'})) {
+ next if (grep {$_ eq $item->{'site'}} @seen);
+ printf "%-7s %-45s %-35s %s\n", $item->{'action'}, $item->{'page'}, $item->{'policy'}, $item->{'category'};
+ push @seen, $item->{'site'};
+ }
+ }
+
+}
+
+say "Running in $LOGMODE mode..." if $LOGMODE;
+netskope();
+say "Completed." if $LOGMODE;
blob - /dev/null
blob + 630f8bac8c7b5a82f3d1c6004929dbcdf0c97dba (mode 755)
--- /dev/null
+++ z.pl
+#!/usr/bin/env perl
+#
+# Copyright 2020, Mischa Peters <mischa AT netskope DOT com>, Netskope.
+# Netskope_ZScalerImporter.pl - Version 3.0 - 20200615
+#
+# Permission to use, copy, modify, and distribute this software for any
+# purpose with or without fee is hereby granted, provided that the above
+# copyright notice and this permission notice appear in all copies.
+#
+# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+#
+# ZScaler integration with Netskope
+#
+use 5.024;
+use strict;
+use warnings;
+use autodie;
+use Config::Tiny;
+use Time::HiRes qw(gettimeofday);
+use POSIX qw(strftime);
+use HTTP::Tiny;
+use HTTP::CookieJar;
+use JSON::PP;
+use Text::CSV;
+use MIME::Lite;
+
+my $LOGMODE = "";
+my @CONFIG_FILES = grep { -e } ('./netskope.cnf', './.netskope.cnf', '/etc/netskope.cnf', "$ENV{'HOME'}/.netskope.cnf", "$ENV{'HOME'}/netskope.cnf");
+my $config = Config::Tiny->read($CONFIG_FILES[-1], 'utf8');
+my $USER_COUNT = $config->{report}{USER_COUNT};
+my $MAX_DOMAIN= $config->{report}{MAX_DOMAIN};
+my $NTSKP_TENANT = $config->{netskope}{NTSKP_TENANT};
+my $NTSKP_TOKEN = $config->{netskope}{NTSKP_TOKEN};
+my $NTSKP_REPORTID = $config->{netskope}{NTSKP_REPORTID};
+my $ZS_MAX_DOMAINS = $config->{zscaler}{ZS_MAX_DOMAINS};
+my $ZS_BASE_URI = $config->{zscaler}{ZS_BASE_URI};
+my $ZS_API_KEY = $config->{zscaler}{ZS_API_KEY};
+my $ZS_API_USERNAME = $config->{zscaler}{ZS_API_USERNAME};
+my $ZS_API_PASSWORD = $config->{zscaler}{ZS_API_PASSWORD};
+my $ZS_CATEGORY_NAME = $config->{zscaler}{ZS_CATEGORY_NAME};
+my $ZS_CATEGORY_DESC = $config->{zscaler}{ZS_CATEGORY_DESC};
+my $PROXY = $config->{general}{PROXY};
+my $SMTP = $config->{general}{SMTP};
+my $FROM = $config->{general}{FROM};
+my $TO = $config->{general}{TO};
+my $SUBJECT = $config->{general}{SUBJECT};
+my $TEXT = $config->{general}{TEXT} . "\n\n";
+my %HEADERS = ("Content-Type" => "application/json", "Cache-Control" => "no-cache");
+my $EMAIL_CSV = "";
+
+### Netskope ###
+sub mail_csv {
+ my $msg = MIME::Lite->new(From=> $FROM, To => $TO, Subject => $SUBJECT, Type => 'TEXT', Data => $TEXT);
+ $msg->attach(Type => 'text/csv', Data => $EMAIL_CSV);
+ $msg->send('smtp', $SMTP, Debug=>0);
+ say "MAIL From: $FROM -> $TO - Attach CSV" if $LOGMODE;
+}
+
+sub _check_return {
+ my ($status, $content, $uri) = @_;
+ if ($status =~ /^2/ && $LOGMODE) {
+ print "URI: $uri\nHTTP RESPONSE: $status\n";
+ print "CONTENT:\n$content\n" if ($LOGMODE eq "DEBUG");
+ }
+ if ($status !~ /^2/) {
+ print "URI: $uri\nHTTP RESPONSE: $status\n$content\n";
+ my $msg = MIME::Lite->new(From => $FROM, To => $TO, Subject => 'API Error along the way', Type => 'TEXT',
+ Data => "URI: $uri\nHTTP RESPONSE: $status\n$content\n\n",
+ );
+ $msg->attach(Type => 'text/csv', Data => $EMAIL_CSV);
+ $msg->send('smtp', $SMTP, Debug=>0);
+ say "MAIL From: $FROM -> $TO - Error Notification" if $LOGMODE;
+ say "exit 1";
+ exit 1;
+ }
+}
+
+sub netskope {
+ ### Collect widget IDs
+ my $uri = "$NTSKP_TENANT/api/v1/reports?token=$NTSKP_TOKEN&op=reportInfo&id=$NTSKP_REPORTID";
+ my $request = HTTP::Tiny->new('default_headers' => \%HEADERS);
+ my $response = $request->get($uri);
+ _check_return($response->{'status'}, $response->{'content'}, $uri);
+
+ my $json = JSON::PP->new->utf8->decode($response->{'content'});
+ my $data = $json->{'data'}->{'latestScheduledRunInfo'}->{'widgets'};
+ if (!$data) { _check_return(404, $response->{'content'}, "No Widget Data"); }
+ my %csv_content;
+
+ ### Collect widget data and write to CSV
+ for my $widget (@{$data}) {
+ $uri = "$NTSKP_TENANT/api/v1/reports?token=$NTSKP_TOKEN&op=widgetData&id=$widget->{'id'}";
+ $response = $request->get($uri);
+ print "\n#DEBUG#\nWidget Name: " . $widget->{'name'} . "\nWidgetID: " . $widget->{'id'} . " (ReportID: $NTSKP_REPORTID)\n" . $response->{'content'} if ($LOGMODE && $LOGMODE eq "DEBUG");
+ _check_return($response->{'status'}, $response->{'content'}, $uri);
+ $csv_content{$widget->{'name'}} = $response->{'content'};
+ }
+
+ ### Process domains from CSV
+ my @blocklist;
+ for my $widget_name (keys %csv_content) {
+ my $count = 0;
+ my $csv = Text::CSV->new({binary => 1, auto_diag => 1});
+ open my $fh_in, "<", \$csv_content{$widget_name};
+ $csv->column_names($csv->getline($fh_in));
+ # "Application","Domain","Category","CCI","Users"
+
+ print "\n## Widget Name: $widget_name\n## Domains: " if $LOGMODE;
+ $EMAIL_CSV .= "$widget_name\n";
+ DOMAIN:
+ while (my $row = $csv->getline_hr($fh_in)) {
+ last DOMAIN if ($count == $MAX_DOMAIN);
+ if ($row->{'Users'} < $USER_COUNT) {
+ print "$row->{'Domain'}," if ($LOGMODE ne "DEBUG");
+ print "$row->{'Application'} - $row->{'Domain'} - $row->{'Category'}, $row->{'CCI'}, $row->{'Users'}\n" if ($LOGMODE eq "DEBUG");
+ $EMAIL_CSV .= "$row->{'Domain'},";
+ push @blocklist, $row->{'Domain'};
+ $count++;
+ }
+ }
+ print "\n\n" if $LOGMODE;
+ $EMAIL_CSV .= "\n";
+ }
+ return @blocklist;
+}
+
+#sub updateUrlList {
+ #my @domains = @{$_[0]};
+ #my $uri = "$NTSKP_TENANT/api/v1/updateUrlList?token=$NTSKP_TOKEN";
+ #my $request = HTTP::Tiny->new('default_headers' => \%HEADERS);
+ #$body = JSON::PP->new->encode({name => $NTSKP_URL_CATEGORY, list => \@domains});
+ #my $response = $request->post($uri, {'content' => $body});
+ #_check_return($response->{'status'}, $response->{'content'}, $uri);
+#}
+
+### Zscaler ###
+
+sub zscaler {
+ my @domains = @{$_[0]};
+
+ ### Authenticate
+ my $now = int(gettimeofday * 1000);
+ my $n = substr($now, -6);
+ my $r = sprintf "%06d", $n >> 1;
+ my $key;
+ for my $i (0..length($n)-1) {
+ $key .= substr($ZS_API_KEY, substr($n, $i, 1), 1);
+ }
+ for my $i (0..length($r)-1) {
+ $key .= substr($ZS_API_KEY, substr($r, $i, 1) + 2, 1);
+ }
+ my $uri = "$ZS_BASE_URI/authenticatedSession";
+ my $body = JSON::PP->new->encode({apiKey => $key, username => $ZS_API_USERNAME, password => $ZS_API_PASSWORD, timestamp => $now});
+ my $jar = HTTP::CookieJar->new;
+ my $request = HTTP::Tiny->new('default_headers' => \%HEADERS, 'cookie_jar' => $jar);
+ my $response = $request->post($uri, {'content' => $body});
+ _check_return($response->{'status'}, $response->{'content'}, $uri);
+
+ ### Get filter list id
+ $uri = "$ZS_BASE_URI/urlCategories/lite";
+ $response = $request->get($uri);
+ _check_return($response->{'status'}, $response->{'content'}, $uri);
+
+ my $json = JSON::PP->new->utf8->decode($response->{'content'});
+ my $id;
+ for my $item (@{$json}) {
+ if (exists($item->{'configuredName'})) {
+ if ($item->{'configuredName'} eq $ZS_CATEGORY_NAME) {
+ $id = $item->{'id'};
+ }
+ }
+ }
+
+ ### Push Domains
+ $uri = defined($id) ? "$ZS_BASE_URI/urlCategories/$id" : "$ZS_BASE_URI/urlCategories";
+ my $method = defined($id) ? "put" : "post";
+ my $description = "$ZS_CATEGORY_DESC\n\nLast Updated: " . strftime("%Y-%m-%d %H:%M:%S", localtime);
+ splice @domains, $ZS_MAX_DOMAINS if @domains > $ZS_MAX_DOMAINS;
+ $body = JSON::PP->new->encode({configuredName => $ZS_CATEGORY_NAME, customCategory => 'true', superCategory => 'SECURITY', urls => \@domains, description => $description});
+ $response = $request->$method($uri, {'content' => $body});
+ _check_return($response->{'status'}, $response->{'content'}, $uri);
+
+ ### Delete authenticadSession
+ $uri = "$ZS_BASE_URI/authenticatedSession";
+ $response = $request->delete($uri);
+ _check_return($response->{'status'}, $response->{'content'}, $uri);
+}
+
+say "Running in $LOGMODE mode..." if $LOGMODE;
+my @domains = netskope();
+zscaler(\@domains);
+mail_csv();
+say "Completed." if $LOGMODE;
blob - /dev/null
blob + 68a5b92b731e8576f3a2cdc02255ac1c073b4443 (mode 755)
--- /dev/null
+++ zscaler-api.pl
+#!/usr/bin/perl
+use 5.024;
+use strict;
+use warnings;
+use autodie;
+use Config::Tiny;
+use Time::HiRes qw(gettimeofday);
+use POSIX qw(strftime);
+use HTTP::Tiny;
+use HTTP::CookieJar;
+use JSON::PP;
+use Text::CSV;
+
+my $verbose = 1;
+my $CONFIG_FILE = "/home/mischa/netskope/netskope.cnf";
+my $config = Config::Tiny->read($CONFIG_FILE, 'utf8');
+my $NTSKP_TENANT = $config->{netskope}{NTSKP_TENANT};
+my $NTSKP_TOKEN = $config->{netskope}{NTSKP_TOKEN};
+my $NTSKP_REPORTID = $config->{netskope}{NTSKP_REPORTID};
+my $ZS_MAX_DOMAINS = $config->{zscaler}{ZS_MAX_DOMAINS};
+my $ZS_BASE_URI = $config->{zscaler}{ZS_BASE_URI};
+my $ZS_API_KEY = $config->{zscaler}{ZS_API_KEY};
+my $ZS_API_USERNAME = $config->{zscaler}{ZS_API_USERNAME};
+my $ZS_API_PASSWORD = $config->{zscaler}{ZS_API_PASSWORD};
+my $ZS_CATEGORY_NAME = $config->{zscaler}{ZS_CATEGORY_NAME};
+my $ZS_CATEGORY_DESC = $config->{zscaler}{ZS_CATEGORY_DESC};
+
+say "Running...";
+
+# Authenticate
+my $now = int(gettimeofday * 1000);
+my $n = substr($now, -6);
+my $r = sprintf "%06d", $n >> 1;
+my $key;
+for my $i (0..length($n)-1) {
+ $key .= substr($ZS_API_KEY, substr($n, $i, 1), 1);
+}
+for my $i (0..length($r)-1) {
+ $key .= substr($ZS_API_KEY, substr($r, $i, 1) + 2, 1);
+}
+my $uri = "$ZS_BASE_URI/authenticatedSession";
+my $body = JSON::PP->new->space_after->encode({apiKey => $key, username => $ZS_API_USERNAME, password => $ZS_API_PASSWORD, timestamp => $now});
+my $jar = HTTP::CookieJar->new;
+my $request = HTTP::Tiny->new('default_headers' => {"Content-Type" => "application/json", "Cache-Control" => "no-cache"}, 'cookie_jar' => $jar);
+my $response = $request->post($uri, {'content' => $body});
+if ($verbose) {
+ say "POST $uri";
+ say "BODY $body";
+ say "HTTP " . $response->{'status'};
+ say "COOKIE " . $jar->cookie_header($ZS_BASE_URI);
+}
+
+# Get filter list id
+$uri = "$ZS_BASE_URI/urlCategories/lite";
+$response = $request->get($uri);
+my $json = JSON::PP->new->utf8->decode($response->{'content'});
+my $id;
+for my $item (@{$json}) {
+ if (exists($item->{'configuredName'})) {
+ if ($item->{'configuredName'} eq $ZS_CATEGORY_NAME) {
+ $id = $item->{'id'};
+ }
+ }
+}
+
+# Push Domains
+my @domains = ('secomtrust.net', 'baidupcs.com', 'cloud.baidu.com');
+$uri = defined($id) ? "$ZS_BASE_URI/urlCategories/$id" : "$ZS_BASE_URI/urlCategories";
+my $method = defined($id) ? "put" : "post";
+my $description = "$ZS_CATEGORY_DESC\n\nLast Updated: " . strftime("%Y-%m-%d %H:%M:%S", localtime);
+$#domains = $#domains >= $ZS_MAX_DOMAINS ? $ZS_MAX_DOMAINS : $#domains;
+$body = JSON::PP->new->space_after->encode({configuredName => $ZS_CATEGORY_NAME, customCategory => 'true', superCategory => 'SECURITY', urls => \@domains, description => $description});
+$response = $request->$method($uri, {'content' => $body});
+if ($verbose) {
+ say uc($method) . " $uri";
+ say "BODY $body";
+ say "HTTP " . $response->{'status'};
+ say "RESPONSE " . $response->{'content'} if ($response->{'status'} =~ /^4/);
+}
+
+# Delete authenticadSession
+$uri = "$ZS_BASE_URI/authenticatedSession";
+$response = $request->delete($uri);
+if ($verbose) {
+ say "DELETE $uri";
+ say "HTTP " . $response->{'status'};
+}
