From abc48b4ed0fc9ba8daca16d0076c13b9d6be1475 Mon Sep 17 00:00:00 2001 From: morpheus65535 Date: Fri, 17 Mar 2023 09:01:15 -0400 Subject: [PATCH] Replaced deprecated Google Universal Analytics by GA4 --- bazarr/app/database.py | 1 - bazarr/app/logger.py | 1 + bazarr/init.py | 3 + bazarr/subtitles/processing.py | 4 +- bazarr/utilities/analytics.py | 89 ++- libs/ga4mp/__init__.py | 3 + libs/ga4mp/event.py | 44 ++ libs/ga4mp/ga4mp.py | 416 +++++++++++++ libs/ga4mp/item.py | 11 + libs/ga4mp/store.py | 116 ++++ libs/ga4mp/utils.py | 392 ++++++++++++ libs/pyga/__init__.py | 8 - libs/pyga/entities.py | 512 ---------------- libs/pyga/exceptions.py | 2 - libs/pyga/requests.py | 1047 -------------------------------- libs/pyga/utils.py | 125 ---- libs/version.txt | 2 +- 17 files changed, 1028 insertions(+), 1748 deletions(-) create mode 100644 libs/ga4mp/__init__.py create mode 100644 libs/ga4mp/event.py create mode 100644 libs/ga4mp/ga4mp.py create mode 100644 libs/ga4mp/item.py create mode 100644 libs/ga4mp/store.py create mode 100644 libs/ga4mp/utils.py delete mode 100644 libs/pyga/__init__.py delete mode 100644 libs/pyga/entities.py delete mode 100644 libs/pyga/exceptions.py delete mode 100644 libs/pyga/requests.py delete mode 100644 libs/pyga/utils.py diff --git a/bazarr/app/database.py b/bazarr/app/database.py index 8f266837b..24fed4d2a 100644 --- a/bazarr/app/database.py +++ b/bazarr/app/database.py @@ -32,7 +32,6 @@ if postgresql: (OperationalError, 'server closed the connection unexpectedly'), ) - logger.debug( f"Connecting to PostgreSQL database: {settings.postgresql.host}:{settings.postgresql.port}/{settings.postgresql.database}") database = ReconnectPostgresqlDatabase(settings.postgresql.database, diff --git a/bazarr/app/logger.py b/bazarr/app/logger.py index f505fe8a2..fc27e0428 100644 --- a/bazarr/app/logger.py +++ b/bazarr/app/logger.py @@ -125,6 +125,7 @@ def configure_logging(debug=False): logging.getLogger("srt").setLevel(logging.ERROR) logging.getLogger("SignalRCoreClient").setLevel(logging.CRITICAL) logging.getLogger("websocket").setLevel(logging.CRITICAL) + logging.getLogger("ga4mp.ga4mp").setLevel(logging.ERROR) logging.getLogger("waitress").setLevel(logging.ERROR) logging.getLogger("knowit").setLevel(logging.CRITICAL) diff --git a/bazarr/init.py b/bazarr/init.py index c1b285970..9fc75f11a 100644 --- a/bazarr/init.py +++ b/bazarr/init.py @@ -57,6 +57,9 @@ os.environ["SZ_HI_EXTENSION"] = settings.general.hi_extension # set anti-captcha provider and key configure_captcha_func() +# import analytics module to make sure logging is properly configured afterwards +from utilities.analytics import event_tracker # noqa E402 + # configure logging configure_logging(settings.general.getboolean('debug') or args.debug) import logging # noqa E402 diff --git a/bazarr/subtitles/processing.py b/bazarr/subtitles/processing.py index c0449efb1..7f7534b5a 100644 --- a/bazarr/subtitles/processing.py +++ b/bazarr/subtitles/processing.py @@ -8,7 +8,7 @@ from utilities.path_mappings import path_mappings from utilities.post_processing import pp_replace, set_chmod from languages.get_languages import alpha2_from_alpha3, alpha2_from_language, alpha3_from_language, language_from_alpha3 from app.database import TableEpisodes, TableMovies -from utilities.analytics import track_event +from utilities.analytics import event_tracker from radarr.notify import notify_radarr from sonarr.notify import notify_sonarr from app.event_handler import event_stream @@ -135,7 +135,7 @@ def process_subtitle(subtitle, media_type, audio_language, path, max_score, is_u notify_radarr(movie_metadata['radarrId']) event_stream(type='movie-wanted', action='delete', payload=movie_metadata['radarrId']) - track_event(category=downloaded_provider, action=action, label=downloaded_language) + event_tracker.track(provider=downloaded_provider, action=action, language=downloaded_language) return ProcessSubtitlesResult(message=message, reversed_path=reversed_path, diff --git a/bazarr/utilities/analytics.py b/bazarr/utilities/analytics.py index 027c6d850..69e5b7a13 100644 --- a/bazarr/utilities/analytics.py +++ b/bazarr/utilities/analytics.py @@ -1,70 +1,59 @@ # coding=utf-8 -import pickle -import random import platform import os import logging -import codecs -from pyga.requests import Event, Tracker, Session, Visitor, Config -from pyga.entities import CustomVariable +from ga4mp import GtagMP from app.get_args import args -from app.config import settings from radarr.info import get_radarr_info from sonarr.info import get_sonarr_info +bazarr_version = os.environ["BAZARR_VERSION"].lstrip('v') +os_version = platform.python_version() sonarr_version = get_sonarr_info.version() radarr_version = get_radarr_info.version() +python_version = platform.platform() -def track_event(category=None, action=None, label=None): - if not settings.analytics.getboolean('enabled'): - return +class EventTracker: + def __init__(self): + self.tracker = GtagMP(api_secret="qHRaseheRsic6-h2I_rIAA", measurement_id="G-3820T18GE3", client_id="temp") - anonymousConfig = Config() - anonymousConfig.anonimize_ip_address = True - - tracker = Tracker('UA-138214134-3', 'none', conf=anonymousConfig) - - try: - if os.path.isfile(os.path.normpath(os.path.join(args.config_dir, 'config', 'analytics.dat'))): - with open(os.path.normpath(os.path.join(args.config_dir, 'config', 'analytics.dat')), 'r') as handle: - visitor_text = handle.read() - visitor = pickle.loads(codecs.decode(visitor_text.encode(), "base64")) - if visitor.user_agent is None: - visitor.user_agent = os.environ.get("SZ_USER_AGENT") - if visitor.unique_id > int(0x7fffffff): - visitor.unique_id = random.randint(0, 0x7fffffff) + if not os.path.isfile(os.path.normpath(os.path.join(args.config_dir, 'config', 'analytics_visitor_id.txt'))): + visitor_id = self.tracker.random_client_id() + with open(os.path.normpath(os.path.join(args.config_dir, 'config', 'analytics_visitor_id.txt')), 'w+') \ + as handle: + handle.write(str(visitor_id)) else: - visitor = Visitor() - visitor.unique_id = random.randint(0, 0x7fffffff) - except Exception: - visitor = Visitor() - visitor.unique_id = random.randint(0, 0x7fffffff) + with open(os.path.normpath(os.path.join(args.config_dir, 'config', 'analytics_visitor_id.txt')), 'r') as \ + handle: + visitor_id = handle.read() - session = Session() - event = Event(category=category, action=action, label=label, value=1) + self.tracker.client_id = visitor_id - tracker.add_custom_variable(CustomVariable(index=1, name='BazarrVersion', - value=os.environ["BAZARR_VERSION"].lstrip('v'), scope=1)) - tracker.add_custom_variable(CustomVariable(index=2, name='PythonVersion', value=platform.python_version(), scope=1)) - if settings.general.getboolean('use_sonarr'): - tracker.add_custom_variable(CustomVariable(index=3, name='SonarrVersion', value=sonarr_version, scope=1)) - else: - tracker.add_custom_variable(CustomVariable(index=3, name='SonarrVersion', value='unused', scope=1)) - if settings.general.getboolean('use_radarr'): - tracker.add_custom_variable(CustomVariable(index=4, name='RadarrVersion', value=radarr_version, scope=1)) - else: - tracker.add_custom_variable(CustomVariable(index=4, name='RadarrVersion', value='unused', scope=1)) - tracker.add_custom_variable(CustomVariable(index=5, name='OSVersion', value=platform.platform(), scope=1)) + self.tracker.store.set_user_property(name="BazarrVersion", value=bazarr_version) + self.tracker.store.set_user_property(name="PythonVersion", value=os_version) + self.tracker.store.set_user_property(name="SonarrVersion", value=sonarr_version) + self.tracker.store.set_user_property(name="RadarrVersion", value=radarr_version) + self.tracker.store.set_user_property(name="OSVersion", value=python_version) - try: - tracker.track_event(event, session, visitor) - except Exception: - logging.debug("BAZARR unable to track event.") - pass - else: - with open(os.path.normpath(os.path.join(args.config_dir, 'config', 'analytics.dat')), 'w+') as handle: - handle.write(codecs.encode(pickle.dumps(visitor), "base64").decode()) + self.tracker.store.save() + + def track(self, provider, action, language): + subtitles_event = self.tracker.create_new_event(name="subtitles") + + subtitles_event.set_event_param(name="subtitles_provider", value=provider) + subtitles_event.set_event_param(name="subtitles_action", value=action) + subtitles_event.set_event_param(name="subtitles_language", value=language) + + try: + self.tracker.send(events=[subtitles_event]) + except Exception: + logging.debug("BAZARR unable to track event.") + else: + self.tracker.store.save() + + +event_tracker = EventTracker() diff --git a/libs/ga4mp/__init__.py b/libs/ga4mp/__init__.py new file mode 100644 index 000000000..9a817b94d --- /dev/null +++ b/libs/ga4mp/__init__.py @@ -0,0 +1,3 @@ +from ga4mp.ga4mp import GtagMP, FirebaseMP + +__all__ = ['GtagMP','FirebaseMP'] \ No newline at end of file diff --git a/libs/ga4mp/event.py b/libs/ga4mp/event.py new file mode 100644 index 000000000..12f65a10a --- /dev/null +++ b/libs/ga4mp/event.py @@ -0,0 +1,44 @@ +from ga4mp.item import Item + +class Event(dict): + def __init__(self, name): + self.set_event_name(name) + + def set_event_name(self, name): + if len(name) > 40: + raise ValueError("Event name cannot exceed 40 characters.") + self["name"] = name + + def get_event_name(self): + return self.get("name") + + def set_event_param(self, name, value): + # Series of checks to comply with GA4 event collection limits: https://support.google.com/analytics/answer/9267744 + if len(name) > 40: + raise ValueError("Event parameter name cannot exceed 40 characters.") + if name in ["page_location", "page_referrer", "page_title"] and len(str(value)) > 300: + raise ValueError("Event parameter value for page info cannot exceed 300 characters.") + if name not in ["page_location", "page_referrer", "page_title"] and len(str(value)) > 100: + raise ValueError("Event parameter value cannot exceed 100 characters.") + if "params" not in self.keys(): + self["params"] = {} + if len(self["params"]) >= 100: + raise RuntimeError("Event cannot contain more than 100 parameters.") + self["params"][name] = value + + def get_event_params(self): + return self.get("params") + + def delete_event_param(self, name): + # Since only 25 event parameters are allowed, this will allow the user to delete a parameter if necessary. + self["params"].pop(name, None) + + def create_new_item(self, item_id=None, item_name=None): + return Item(item_id=item_id, item_name=item_name) + + def add_item_to_event(self, item): + if not isinstance(item, dict): + raise ValueError("'item' must be an instance of a dictionary.") + if "items" not in self["params"].keys(): + self.set_event_param("items", []) + self["params"]["items"].append(item) \ No newline at end of file diff --git a/libs/ga4mp/ga4mp.py b/libs/ga4mp/ga4mp.py new file mode 100644 index 000000000..45fdc842b --- /dev/null +++ b/libs/ga4mp/ga4mp.py @@ -0,0 +1,416 @@ +############################################################################### +# Google Analytics 4 Measurement Protocol for Python +# Copyright (c) 2022, Adswerve +# +# This project is free software, distributed under the BSD license. +# Adswerve offers consulting and integration services if your firm needs +# assistance in strategy, implementation, or auditing existing work. +############################################################################### + +import json +import logging +import urllib.request +import time +import datetime +import random +from ga4mp.utils import params_dict +from ga4mp.event import Event +from ga4mp.store import BaseStore, DictStore + +import os, sys +sys.path.append( + os.path.normpath(os.path.join(os.path.dirname(__file__), "..")) +) + +logger = logging.getLogger(__name__) +logger.setLevel(logging.INFO) + +class BaseGa4mp(object): + """ + Parent class that provides an interface for sending data to Google Analytics, supporting the GA4 Measurement Protocol. + + Parameters + ---------- + api_secret : string + Generated through the Google Analytics UI. To create a new secret, navigate in the Google Analytics UI to: Admin > Data Streams > + [choose your stream] > Measurement Protocol API Secrets > Create + + See Also + -------- + + * Measurement Protocol (Google Analytics 4): https://developers.google.com/analytics/devguides/collection/protocol/ga4 + + Examples + -------- + # Initialize tracking object for gtag usage + >>> ga = gtagMP(api_secret = "API_SECRET", measurement_id = "MEASUREMENT_ID", client_id="CLIENT_ID") + + # Initialize tracking object for Firebase usage + >>> ga = firebaseMP(api_secret = "API_SECRET", firebase_app_id = "FIREBASE_APP_ID", app_instance_id="APP_INSTANCE_ID") + + # Build an event + >>> event_type = 'new_custom_event' + >>> event_parameters = {'parameter_key_1': 'parameter_1', 'parameter_key_2': 'parameter_2'} + >>> event = {'name': event_type, 'params': event_parameters } + >>> events = [event] + + # Send a custom event to GA4 immediately + >>> ga.send(events) + + # Postponed send of a custom event to GA4 + >>> ga.send(events, postpone=True) + >>> ga.postponed_send() + """ + + def __init__(self, api_secret, store: BaseStore = None): + self._initialization_time = time.time() # used for both session_id and calculating engagement time + self.api_secret = api_secret + self._event_list = [] + assert store is None or isinstance(store, BaseStore), "if supplied, store must be an instance of BaseStore" + self.store = store or DictStore() + self._check_store_requirements() + self._base_domain = "https://www.google-analytics.com/mp/collect" + self._validation_domain = "https://www.google-analytics.com/debug/mp/collect" + + def _check_store_requirements(self): + # Store must contain "session_id" and "last_interaction_time_msec" in order for tracking to work properly. + if self.store.get_session_parameter("session_id") is None: + self.store.set_session_parameter(name="session_id", value=int(self._initialization_time)) + # Note: "last_interaction_time_msec" factors into the required "engagement_time_msec" event parameter. + self.store.set_session_parameter(name="last_interaction_time_msec", value=int(self._initialization_time * 1000)) + + def create_new_event(self, name): + return Event(name=name) + + def send(self, events, validation_hit=False, postpone=False, date=None): + """ + Method to send an http post request to google analytics with the specified events. + + Parameters + ---------- + events : List[Dict] + A list of dictionaries of the events to be sent to Google Analytics. The list of dictionaries should adhere + to the following format: + + [{'name': 'level_end', + 'params' : {'level_name': 'First', + 'success': 'True'} + }, + {'name': 'level_up', + 'params': {'character': 'John Madden', + 'level': 'First'} + }] + + validation_hit : bool, optional + Boolean to depict if events should be tested against the Measurement Protocol Validation Server, by default False + postpone : bool, optional + Boolean to depict if provided event list should be postponed, by default False + date : datetime + Python datetime object for sending a historical event at the given date. Date cannot be in the future. + """ + + # check for any missing or invalid parameters among automatically collected and recommended event types + self._check_params(events) + self._check_date_not_in_future(date) + self._add_session_id_and_engagement_time(events) + + if postpone is True: + # build event list to send later + for event in events: + event["_timestamp_micros"] = self._get_timestamp(time.time()) + self._event_list.append(event) + else: + # batch events into sets of 25 events, the maximum allowed. + batched_event_list = [ + events[event : event + 25] for event in range(0, len(events), 25) + ] + # send http post request + self._http_post( + batched_event_list, validation_hit=validation_hit, date=date + ) + + def postponed_send(self): + """ + Method to send the events provided to Ga4mp.send(events,postpone=True) + """ + + for event in self._event_list: + self._http_post([event], postpone=True) + + # clear event_list for future use + self._event_list = [] + + def append_event_to_params_dict(self, new_name_and_parameters): + + """ + Method to append event name and parameters key-value pairing(s) to parameters dictionary. + + Parameters + ---------- + new_name_and_parameters : Dict + A dictionary with one key-value pair representing a new type of event to be sent to Google Analytics. + The dictionary should adhere to the following format: + + {'new_name': ['new_param_1', 'new_param_2', 'new_param_3']} + """ + + params_dict.update(new_name_and_parameters) + + def _http_post(self, batched_event_list, validation_hit=False, postpone=False, date=None): + """ + Method to send http POST request to google-analytics. + + Parameters + ---------- + batched_event_list : List[List[Dict]] + List of List of events. Places initial event payload into a list to send http POST in batches. + validation_hit : bool, optional + Boolean to depict if events should be tested against the Measurement Protocol Validation Server, by default False + postpone : bool, optional + Boolean to depict if provided event list should be postponed, by default False + date : datetime + Python datetime object for sending a historical event at the given date. Date cannot be in the future. + Timestamp micros supports up to 48 hours of backdating. + If date is specified, postpone must be False or an assertion will be thrown. + """ + self._check_date_not_in_future(date) + status_code = None # Default set to know if batch loop does not work and to bound status_code + + # set domain + domain = self._base_domain + if validation_hit is True: + domain = self._validation_domain + logger.info(f"Sending POST to: {domain}") + + # loop through events in batches of 25 + batch_number = 1 + for batch in batched_event_list: + # url and request slightly differ by subclass + url = self._build_url(domain=domain) + request = self._build_request(batch=batch) + self._add_user_props_to_hit(request) + + # make adjustments for postponed hit + request["events"] = ( + {"name": batch["name"], "params": batch["params"]} + if (postpone) + else batch + ) + + if date is not None: + logger.info(f"Setting event timestamp to: {date}") + assert ( + postpone is False + ), "Cannot send postponed historical hit, ensure postpone=False" + + ts = self._datetime_to_timestamp(date) + ts_micro = self._get_timestamp(ts) + request["timestamp_micros"] = int(ts_micro) + logger.info(f"Timestamp of request is: {request['timestamp_micros']}") + + if postpone: + # add timestamp to hit + request["timestamp_micros"] = batch["_timestamp_micros"] + + req = urllib.request.Request(url) + req.add_header("Content-Type", "application/json; charset=utf-8") + jsondata = json.dumps(request) + json_data_as_bytes = jsondata.encode("utf-8") # needs to be bytes + req.add_header("Content-Length", len(json_data_as_bytes)) + result = urllib.request.urlopen(req, json_data_as_bytes) + + status_code = result.status + logger.info(f"Batch Number: {batch_number}") + logger.info(f"Status code: {status_code}") + batch_number += 1 + + return status_code + + def _check_params(self, events): + + """ + Method to check whether the provided event payload parameters align with supported parameters. + + Parameters + ---------- + events : List[Dict] + A list of dictionaries of the events to be sent to Google Analytics. The list of dictionaries should adhere + to the following format: + + [{'name': 'level_end', + 'params' : {'level_name': 'First', + 'success': 'True'} + }, + {'name': 'level_up', + 'params': {'character': 'John Madden', + 'level': 'First'} + }] + """ + + # check to make sure it's a list of dictionaries with the right keys + + assert type(events) == list, "events should be a list" + + for event in events: + + assert isinstance(event, dict), "each event should be an instance of a dictionary" + + assert "name" in event, 'each event should have a "name" key' + + assert "params" in event, 'each event should have a "params" key' + + # check for any missing or invalid parameters + + for e in events: + event_name = e["name"] + event_params = e["params"] + if event_name in params_dict.keys(): + for parameter in params_dict[event_name]: + if parameter not in event_params.keys(): + logger.warning( + f"WARNING: Event parameters do not match event type.\nFor {event_name} event type, the correct parameter(s) are {params_dict[event_name]}.\nThe parameter '{parameter}' triggered this warning.\nFor a breakdown of currently supported event types and their parameters go here: https://support.google.com/analytics/answer/9267735\n" + ) + + def _add_session_id_and_engagement_time(self, events): + """ + Method to add the session_id and engagement_time_msec parameter to all events. + """ + for event in events: + current_time_in_milliseconds = int(time.time() * 1000) + + event_params = event["params"] + if "session_id" not in event_params.keys(): + event_params["session_id"] = self.store.get_session_parameter("session_id") + if "engagement_time_msec" not in event_params.keys(): + last_interaction_time = self.store.get_session_parameter("last_interaction_time_msec") + event_params["engagement_time_msec"] = current_time_in_milliseconds - last_interaction_time if current_time_in_milliseconds > last_interaction_time else 0 + self.store.set_session_parameter(name="last_interaction_time_msec", value=current_time_in_milliseconds) + + def _add_user_props_to_hit(self, hit): + + """ + Method is a helper function to add user properties to outgoing hits. + + Parameters + ---------- + hit : dict + """ + + for key in self.store.get_all_user_properties(): + try: + if key in ["user_id", "non_personalized_ads"]: + hit.update({key: self.store.get_user_property(key)}) + else: + if "user_properties" not in hit.keys(): + hit.update({"user_properties": {}}) + hit["user_properties"].update( + {key: {"value": self.store.get_user_property(key)}} + ) + except: + logger.info(f"Failed to add user property to outgoing hit: {key}") + + def _get_timestamp(self, timestamp): + """ + Method returns UNIX timestamp in microseconds for postponed hits. + + Parameters + ---------- + None + """ + return int(timestamp * 1e6) + + def _datetime_to_timestamp(self, dt): + """ + Private method to convert a datetime object into a timestamp + + Parameters + ---------- + dt : datetime + A datetime object in any format + + Returns + ------- + timestamp + A UNIX timestamp in milliseconds + """ + return time.mktime(dt.timetuple()) + + def _check_date_not_in_future(self, date): + """ + Method to check that provided date is not in the future. + + Parameters + ---------- + date : datetime + Python datetime object + """ + if date is None: + pass + else: + assert ( + date <= datetime.datetime.now() + ), "Provided date cannot be in the future" + + def _build_url(self, domain): + raise NotImplementedError("Subclass should be using this function, but it was called through the base class instead.") + + def _build_request(self, batch): + raise NotImplementedError("Subclass should be using this function, but it was called through the base class instead.") + +class GtagMP(BaseGa4mp): + """ + Subclass for users of gtag. See `Ga4mp` parent class for examples. + + Parameters + ---------- + measurement_id : string + The identifier for a Data Stream. Found in the Google Analytics UI under: Admin > Data Streams > [choose your stream] > Measurement ID (top-right) + client_id : string + A unique identifier for a client, representing a specific browser/device. + """ + + def __init__(self, api_secret, measurement_id, client_id,): + super().__init__(api_secret) + self.measurement_id = measurement_id + self.client_id = client_id + + def _build_url(self, domain): + return f"{domain}?measurement_id={self.measurement_id}&api_secret={self.api_secret}" + + def _build_request(self, batch): + return {"client_id": self.client_id, "events": batch} + + def random_client_id(self): + """ + Utility function for generating a new client ID matching the typical format of 10 random digits and the UNIX timestamp in seconds, joined by a period. + """ + return "%0.10d" % random.randint(0,9999999999) + "." + str(int(time.time())) + +class FirebaseMP(BaseGa4mp): + """ + Subclass for users of Firebase. See `Ga4mp` parent class for examples. + + Parameters + ---------- + firebase_app_id : string + The identifier for a Firebase app. Found in the Firebase console under: Project Settings > General > Your Apps > App ID. + app_instance_id : string + A unique identifier for a Firebase app instance. + * Android - getAppInstanceId() - https://firebase.google.com/docs/reference/android/com/google/firebase/analytics/FirebaseAnalytics#public-taskstring-getappinstanceid + * Kotlin - getAppInstanceId() - https://firebase.google.com/docs/reference/kotlin/com/google/firebase/analytics/FirebaseAnalytics#getappinstanceid + * Swift - appInstanceID() - https://firebase.google.com/docs/reference/swift/firebaseanalytics/api/reference/Classes/Analytics#appinstanceid + * Objective-C - appInstanceID - https://firebase.google.com/docs/reference/ios/firebaseanalytics/api/reference/Classes/FIRAnalytics#+appinstanceid + * C++ - GetAnalyticsInstanceId() - https://firebase.google.com/docs/reference/cpp/namespace/firebase/analytics#getanalyticsinstanceid + * Unity - GetAnalyticsInstanceIdAsync() - https://firebase.google.com/docs/reference/unity/class/firebase/analytics/firebase-analytics#getanalyticsinstanceidasync + """ + + def __init__(self, api_secret, firebase_app_id, app_instance_id): + super().__init__(api_secret) + self.firebase_app_id = firebase_app_id + self.app_instance_id = app_instance_id + + def _build_url(self, domain): + return f"{domain}?firebase_app_id={self.firebase_app_id}&api_secret={self.api_secret}" + + def _build_request(self, batch): + return {"app_instance_id": self.app_instance_id, "events": batch} \ No newline at end of file diff --git a/libs/ga4mp/item.py b/libs/ga4mp/item.py new file mode 100644 index 000000000..9c5ee9cd9 --- /dev/null +++ b/libs/ga4mp/item.py @@ -0,0 +1,11 @@ +class Item(dict): + def __init__(self, item_id=None, item_name=None): + if item_id is None and item_name is None: + raise ValueError("At least one of 'item_id' and 'item_name' is required.") + if item_id is not None: + self.set_parameter("item_id", str(item_id)) + if item_name is not None: + self.set_parameter("item_name", item_name) + + def set_parameter(self, name, value): + self[name] = value \ No newline at end of file diff --git a/libs/ga4mp/store.py b/libs/ga4mp/store.py new file mode 100644 index 000000000..d85bc0c2a --- /dev/null +++ b/libs/ga4mp/store.py @@ -0,0 +1,116 @@ +import json +import logging +from pathlib import Path + +logger = logging.getLogger(__name__) +logger.setLevel(logging.INFO) + +class BaseStore(dict): + def __init__(self): + self.update([("user_properties", {}),("session_parameters", {})]) + + def save(self): + raise NotImplementedError("Subclass should be using this function, but it was called through the base class instead.") + + def _check_exists(self, key): + # Helper function to make sure a key exists before trying to work with values within it. + if key not in self.keys(): + self[key] = {} + + def _set(self, param_type, name, value): + # Helper function to set a single parameter (user or session or other). + self._check_exists(key=param_type) + self[param_type][name] = value + + def _get_one(self, param_type, name): + # Helper function to get a single parameter value (user or session). + self._check_exists(key=param_type) + return self[param_type].get(name, None) + + def _get_all(self, param_type=None): + # Helper function to get all user or session parameters - or the entire dictionary if not specified. + if param_type is not None: + return self[param_type] + else: + return self + + # While redundant, the following make sure the distinction between session and user items is easier for the end user. + def set_user_property(self, name, value): + self._set(param_type="user_properties", name=name, value=value) + + def get_user_property(self, name): + return self._get_one(param_type="user_properties", name=name) + + def get_all_user_properties(self): + return self._get_all(param_type="user_properties") + + def clear_user_properties(self): + self["user_properties"] = {} + + def set_session_parameter(self, name, value): + self._set(param_type="session_parameters", name=name, value=value) + + def get_session_parameter(self, name): + return self._get_one(param_type="session_parameters", name=name) + + def get_all_session_parameters(self): + return self._get_all(param_type="session_parameters") + + def clear_session_parameters(self): + self["session_parameters"] = {} + + # Similar functions for other items the user wants to store that don't fit the other two categories. + def set_other_parameter(self, name, value): + self._set(param_type="other", name=name, value=value) + + def get_other_parameter(self, name): + return self._get_one(param_type="other", name=name) + + def get_all_other_parameters(self): + return self._get_all(param_type="other") + + def clear_other_parameters(self): + self["other"] = {} + +class DictStore(BaseStore): + # Class for working with dictionaries that persist for the life of the class. + def __init__(self, data: dict = None): + super().__init__() + if data: + self.update(data) + + def save(self): + # Give the user back what's in the dictionary so they can decide how to save it. + self._get_all() + +class FileStore(BaseStore): + # Class for working with dictionaries that get saved to a JSON file. + def __init__(self, data_location: str = None): + super().__init__() + self.data_location = data_location + try: + self._load_file(data_location) + except: + logger.info(f"Failed to find file at location: {data_location}") + + def _load_file(self): + # Function to get data from the object's initialized location. + # If the provided or stored data_location exists, read the file and overwrite the object's contents. + if Path(self.data_location).exists(): + with open(self.data_location, "r") as json_file: + self = json.load(json_file) + # If the data_location doesn't exist, try to create a new starter JSON file at the location given. + else: + starter_dict = '{"user_properties":{}, "session_parameters":{}}' + starter_json = json.loads(starter_dict) + Path(self.data_location).touch() + with open(self.data_location, "w") as json_file: + json.dumps(starter_json, json_file) + + def save(self): + # Function to save the current dictionary to a JSON file at the object's initialized location. + try: + with open(self.data_location, "w") as outfile: + json.dump(self, outfile) + except: + logger.info(f"Failed to save file at location: {self.data_location}") \ No newline at end of file diff --git a/libs/ga4mp/utils.py b/libs/ga4mp/utils.py new file mode 100644 index 000000000..27fbca86c --- /dev/null +++ b/libs/ga4mp/utils.py @@ -0,0 +1,392 @@ +# all automatically collected and recommended event types +params_dict = { + "ad_click": [ + "ad_event_id" + ], + "ad_exposure": [ + "firebase_screen", + "firebase_screen_id", + "firebase_screen_class", + "exposure_time", + ], + "ad_impression": [ + "ad_event_id" + ], + "ad_query": [ + "ad_event_id" + ], + "ad_reward": [ + "ad_unit_id", + "reward_type", + "reward_value" + ], + "add_payment_info": [ + "coupon", + "currency", + "items", + "payment_type", + "value" + ], + "add_shipping_info": [ + "coupon", + "currency", + "items", + "shipping_tier", + "value" + ], + "add_to_cart": [ + "currency", + "items", + "value" + ], + "add_to_wishlist": [ + "currency", + "items", + "value" + ], + "adunit_exposure": [ + "firebase_screen", + "firebase_screen_id", + "firebase_screen_class", + "exposure_time", + ], + "app_clear_data": [], + "app_exception": [ + "fatal", + "timestamp", + "engagement_time_msec" + ], + "app_remove": [], + "app_store_refund": [ + "product_id", + "value", + "currency", + "quantity" + ], + "app_store_subscription_cancel": [ + "product_id", + "price", + "value", + "currency", + "cancellation_reason", + ], + "app_store_subscription_convert": [ + "product_id", + "price", + "value", + "currency", + "quantity", + ], + "app_store_subscription_renew": [ + "product_id", + "price", + "value", + "currency", + "quantity", + "renewal_count", + ], + "app_update": [ + "previous_app_version" + ], + "begin_checkout": [ + "coupon", + "currency", + "items", + "value" + ], + "click": [], + "dynamic_link_app_open": [ + "source", + "medium", + "campaign", + "link_id", + "accept_time" + ], + "dynamic_link_app_update": [ + "source", + "medium", + "campaign", + "link_id", + "accept_time", + ], + "dynamic_link_first_open": [ + "source", + "medium", + "campaign", + "link_id", + "accept_time", + ], + "earn_virtual_currency": [ + "virtual_currency_name", + "value" + ], + "error": [ + "firebase_error", + "firebase_error_value" + ], + "file_download": [ + "file_extension", + "file_name", + "link_classes", + "link_domain", + "link_id", + "link_text", + "link_url", + ], + "firebase_campaign": [ + "source", + "medium", + "campaign", + "term", + "content", + "gclid", + "aclid", + "cp1", + "anid", + "click_timestamp", + "campaign_info_source", + ], + "firebase_in_app_message_action": [ + "message_name", + "message_device_time", + "message_id", + ], + "firebase_in_app_message_dismiss": [ + "message_name", + "message_device_time", + "message_id", + ], + "firebase_in_app_message_impression": [ + "message_name", + "message_device_time", + "message_id", + ], + "first_open": [ + "previous_gmp_app_id", + "updated_with_analytics", + "previous_first_open_count", + "system_app", + "system_app_update", + "deferred_analytics_collection", + "reset_analytics_cause", + "engagement_time_msec", + ], + "first_visit": [], + "generate_lead": [ + "value", + "currency" + ], + "in_app_purchase": [ + "product_id", + "price", + "value", + "currency", + "quantity", + "subscription", + "free_trial", + "introductory_price", + ], + "join_group": [ + "group_id" + ], + "level_end": [ + "level_name", + "success" + ], + "level_start": [ + "level_name" + ], + "level_up": [ + "character", + "level" + ], + "login": [ + "method" + ], + "notification_dismiss": [ + "message_name", + "message_time", + "message_device_time", + "message_id", + "topic", + "label", + "message_channel", + ], + "notification_foreground": [ + "message_name", + "message_time", + "message_device_time", + "message_id", + "topic", + "label", + "message_channel", + "message_type", + ], + "notification_open": [ + "message_name", + "message_time", + "message_device_time", + "message_id", + "topic", + "label", + "message_channel", + ], + "notification_receive": [ + "message_name", + "message_time", + "message_device_time", + "message_id", + "topic", + "label", + "message_channel", + "message_type", + ], + "notification_send": [ + "message_name", + "message_time", + "message_device_time", + "message_id", + "topic", + "label", + "message_channel", + ], + "os_update": [ + "previous_os_version" + ], + "page_view": [ + "page_location", + "page_referrer" + ], + "post_score": [ + "level", + "character", + "score" + ], + "purchase": [ + "affiliation", + "coupon", + "currency", + "items", + "transaction_id", + "shipping", + "tax", + "value", + ], + "refund": [ + "transaction_id", + "value", + "currency", + "tax", + "shipping", + "items" + ], + "remove_from_cart": [ + "currency", + "items", + "value" + ], + "screen_view": [ + "firebase_screen", + "firebase_screen_class", + "firebase_screen_id", + "firebase_previous_screen", + "firebase_previous_class", + "firebase_previous_id", + "engagement_time_msec", + ], + "scroll": [], + "search": [ + "search_term" + ], + "select_content": [ + "content_type", + "item_id" + ], + "select_item": [ + "items", + "item_list_name", + "item_list_id" + ], + "select_promotion": [ + "items", + "promotion_id", + "promotion_name", + "creative_name", + "creative_slot", + "location_id", + ], + "session_start": [], + "share": [ + "content_type", + "item_id" + ], + "sign_up": [ + "method" + ], + "view_search_results": [ + "search_term" + ], + "spend_virtual_currency": [ + "item_name", + "virtual_currency_name", + "value" + ], + "tutorial_begin": [], + "tutorial_complete": [], + "unlock_achievement": [ + "achievement_id" + ], + "user_engagement": [ + "engagement_time_msec" + ], + "video_start": [ + "video_current_time", + "video_duration", + "video_percent", + "video_provider", + "video_title", + "video_url", + "visible", + ], + "video_progress": [ + "video_current_time", + "video_duration", + "video_percent", + "video_provider", + "video_title", + "video_url", + "visible", + ], + "video_complete": [ + "video_current_time", + "video_duration", + "video_percent", + "video_provider", + "video_title", + "video_url", + "visible", + ], + "view_cart": [ + "currency", + "items", + "value" + ], + "view_item": [ + "currency", + "items", + "value" + ], + "view_item_list": [ + "items", + "item_list_name", + "item_list_id" + ], + "view_promotion": [ + "items", + "promotion_id", + "promotion_name", + "creative_name", + "creative_slot", + "location_id", + ], +} \ No newline at end of file diff --git a/libs/pyga/__init__.py b/libs/pyga/__init__.py deleted file mode 100644 index 103d0ccbf..000000000 --- a/libs/pyga/__init__.py +++ /dev/null @@ -1,8 +0,0 @@ -from pyga.requests import Q - -def shutdown(): - ''' - Fire all stored GIF requests One by One. - You should call this if you set Config.queue_requests = True - ''' - map(lambda func: func(), Q.REQ_ARRAY) diff --git a/libs/pyga/entities.py b/libs/pyga/entities.py deleted file mode 100644 index 2c049427f..000000000 --- a/libs/pyga/entities.py +++ /dev/null @@ -1,512 +0,0 @@ -# -*- coding: utf-8 -*- - -from datetime import datetime -from operator import itemgetter -from pyga import utils -from pyga import exceptions -try: - from urlparse import urlparse - from urllib import unquote_plus -except ImportError as e: - from urllib.parse import urlparse - from urllib.parse import unquote_plus - - -__author__ = "Arun KR (kra3) " -__license__ = "Simplified BSD" - - -class Campaign(object): - ''' - A representation of Campaign - - Properties: - _type -- See TYPE_* constants, will be mapped to "__utmz" parameter. - creation_time -- Time of the creation of this campaign, will be mapped to "__utmz" parameter. - response_count -- Response Count, will be mapped to "__utmz" parameter. - Is also used to determine whether the campaign is new or repeated, - which will be mapped to "utmcn" and "utmcr" parameters. - id -- Campaign ID, a.k.a. "utm_id" query parameter for ga.js - Will be mapped to "__utmz" parameter. - source -- Source, a.k.a. "utm_source" query parameter for ga.js. - Will be mapped to "utmcsr" key in "__utmz" parameter. - g_click_id -- Google AdWords Click ID, a.k.a. "gclid" query parameter for ga.js. - Will be mapped to "utmgclid" key in "__utmz" parameter. - d_click_id -- DoubleClick (?) Click ID. Will be mapped to "utmdclid" key in "__utmz" parameter. - name -- Name, a.k.a. "utm_campaign" query parameter for ga.js. - Will be mapped to "utmccn" key in "__utmz" parameter. - medium -- Medium, a.k.a. "utm_medium" query parameter for ga.js. - Will be mapped to "utmcmd" key in "__utmz" parameter. - term -- Terms/Keywords, a.k.a. "utm_term" query parameter for ga.js. - Will be mapped to "utmctr" key in "__utmz" parameter. - content -- Ad Content Description, a.k.a. "utm_content" query parameter for ga.js. - Will be mapped to "utmcct" key in "__utmz" parameter. - - ''' - - TYPE_DIRECT = 'direct' - TYPE_ORGANIC = 'organic' - TYPE_REFERRAL = 'referral' - - CAMPAIGN_DELIMITER = '|' - - UTMZ_PARAM_MAP = { - 'utmcid': 'id', - 'utmcsr': 'source', - 'utmgclid': 'g_click_id', - 'utmdclid': 'd_click_id', - 'utmccn': 'name', - 'utmcmd': 'medium', - 'utmctr': 'term', - 'utmcct': 'content', - } - - def __init__(self, typ): - self._type = None - self.creation_time = None - self.response_count = 0 - self.id = None - self.source = None - self.g_click_id = None - self.d_click_id = None - self.name = None - self.medium = None - self.term = None - self.content = None - - if typ: - if typ not in ('direct', 'organic', 'referral'): - raise ValueError('Campaign type has to be one of the Campaign::TYPE_* constant values.') - - self._type = typ - if typ == Campaign.TYPE_DIRECT: - self.name = '(direct)' - self.source = '(direct)' - self.medium = '(none)' - elif typ == Campaign.TYPE_REFERRAL: - self.name = '(referral)' - self.medium = 'referral' - elif typ == Campaign.TYPE_ORGANIC: - self.name = '(organic)' - self.medium = 'organic' - else: - self._type = None - - self.creation_time = datetime.utcnow() - - def validate(self): - if not self.source: - raise exceptions.ValidationError('Campaigns need to have at least the "source" attribute defined.') - - @staticmethod - def create_from_referrer(url): - obj = Campaign(Campaign.TYPE_REFERRAL) - parse_rslt = urlparse(url) - obj.source = parse_rslt.netloc - obj.content = parse_rslt.path - return obj - - def extract_from_utmz(self, utmz): - parts = utmz.split('.', 4) - - if len(parts) != 5: - raise ValueError('The given "__utmz" cookie value is invalid.') - - self.creation_time = utils.convert_ga_timestamp(parts[1]) - self.response_count = int(parts[3]) - params = parts[4].split(Campaign.CAMPAIGN_DELIMITER) - - for param in params: - key, val = param.split('=') - - try: - setattr(self, self.UTMZ_PARAM_MAP[key], unquote_plus(val)) - except KeyError: - continue - - return self - - -class CustomVariable(object): - ''' - Represent a Custom Variable - - Properties: - index -- Is the slot, you have 5 slots - name -- Name given to custom variable - value -- Value for the variable - scope -- Scope can be any one of 1, 2 or 3. - - WATCH OUT: It's a known issue that GA will not decode URL-encoded - characters in custom variable names and values properly, so spaces - will show up as "%20" in the interface etc. (applicable to name & value) - http://www.google.com/support/forum/p/Google%20Analytics/thread?tid=2cdb3ec0be32e078 - - ''' - - SCOPE_VISITOR = 1 - SCOPE_SESSION = 2 - SCOPE_PAGE = 3 - - def __init__(self, index=None, name=None, value=None, scope=3): - self.index = index - self.name = name - self.value = value - self.scope = CustomVariable.SCOPE_PAGE - if scope: - self.scope = scope - - def __setattr__(self, name, value): - if name == 'scope': - if value and value not in range(1, 4): - raise ValueError('Custom Variable scope has to be one of the 1,2 or 3') - - if name == 'index': - # Custom Variables are limited to five slots officially, but there seems to be a - # trick to allow for more of them which we could investigate at a later time (see - # http://analyticsimpact.com/2010/05/24/get-more-than-5-custom-variables-in-google-analytics/ - if value and (value < 0 or value > 5): - raise ValueError('Custom Variable index has to be between 1 and 5.') - - object.__setattr__(self, name, value) - - def validate(self): - ''' - According to the GA documentation, there is a limit to the combined size of - name and value of 64 bytes after URL encoding, - see http://code.google.com/apis/analytics/docs/tracking/gaTrackingCustomVariables.html#varTypes - and http://xahlee.org/js/google_analytics_tracker_2010-07-01_expanded.js line 563 - This limit was increased to 128 bytes BEFORE encoding with the 2012-01 release of ga.js however, - see http://code.google.com/apis/analytics/community/gajs_changelog.html - ''' - if len('%s%s' % (self.name, self.value)) > 128: - raise exceptions.ValidationError('Custom Variable combined name and value length must not be larger than 128 bytes.') - - -class Event(object): - ''' - Represents an Event - https://developers.google.com/analytics/devguides/collection/gajs/eventTrackerGuide - - Properties: - category -- The general event category - action -- The action for the event - label -- An optional descriptor for the event - value -- An optional value associated with the event. You can see your - event values in the Overview, Categories, and Actions reports, - where they are listed by event or aggregated across events, - depending upon your report view. - noninteraction -- By default, event hits will impact a visitor's bounce rate. - By setting this parameter to true, this event hit - will not be used in bounce rate calculations. - (default False) - ''' - - def __init__(self, category=None, action=None, label=None, value=None, noninteraction=False): - self.category = category - self.action = action - self.label = label - self.value = value - self.noninteraction = bool(noninteraction) - - if self.noninteraction and not self.value: - self.value = 0 - - def validate(self): - if not(self.category and self.action): - raise exceptions.ValidationError('Events, at least need to have a category and action defined.') - - -class Item(object): - ''' - Represents an Item in Transaction - - Properties: - order_id -- Order ID, will be mapped to "utmtid" parameter - sku -- Product Code. This is the sku code for a given product, will be mapped to "utmipc" parameter - name -- Product Name, will be mapped to "utmipn" parameter - variation -- Variations on an item, will be mapped to "utmiva" parameter - price -- Unit Price. Value is set to numbers only, will be mapped to "utmipr" parameter - quantity -- Unit Quantity, will be mapped to "utmiqt" parameter - - ''' - - def __init__(self): - self.order_id = None - self.sku = None - self.name = None - self.variation = None - self.price = None - self.quantity = 1 - - def validate(self): - if not self.sku: - raise exceptions.ValidationError('sku/product is a required parameter') - - -class Page(object): - ''' - Contains all parameters needed for tracking a page - - Properties: - path -- Page request URI, will be mapped to "utmp" parameter - title -- Page title, will be mapped to "utmdt" parameter - charset -- Charset encoding, will be mapped to "utmcs" parameter - referrer -- Referer URL, will be mapped to "utmr" parameter - load_time -- Page load time in milliseconds, will be encoded into "utme" parameter. - - ''' - REFERRER_INTERNAL = '0' - - def __init__(self, path): - self.path = None - self.title = None - self.charset = None - self.referrer = None - self.load_time = None - - if path: - self.path = path - - def __setattr__(self, name, value): - if name == 'path': - if value and value != '': - if value[0] != '/': - raise ValueError('The page path should always start with a slash ("/").') - elif name == 'load_time': - if value and not isinstance(value, int): - raise ValueError('Page load time must be specified in integer milliseconds.') - - object.__setattr__(self, name, value) - - -class Session(object): - ''' - You should serialize this object and store it in the user session to keep it - persistent between requests (similar to the "__umtb" cookie of the GA Javascript client). - - Properties: - session_id -- A unique per-session ID, will be mapped to "utmhid" parameter - track_count -- The amount of pageviews that were tracked within this session so far, - will be part of the "__utmb" cookie parameter. - Will get incremented automatically upon each request - start_time -- Timestamp of the start of this new session, will be part of the "__utmb" cookie parameter - - ''' - def __init__(self): - self.session_id = utils.get_32bit_random_num() - self.track_count = 0 - self.start_time = datetime.utcnow() - - @staticmethod - def generate_session_id(): - return utils.get_32bit_random_num() - - def extract_from_utmb(self, utmb): - ''' - Will extract information for the "trackCount" and "startTime" - properties from the given "__utmb" cookie value. - ''' - parts = utmb.split('.') - if len(parts) != 4: - raise ValueError('The given "__utmb" cookie value is invalid.') - - self.track_count = int(parts[1]) - self.start_time = utils.convert_ga_timestamp(parts[3]) - - return self - - -class SocialInteraction(object): - ''' - - Properties: - action -- Required. A string representing the social action being tracked, - will be mapped to "utmsa" parameter - network -- Required. A string representing the social network being tracked, - will be mapped to "utmsn" parameter - target -- Optional. A string representing the URL (or resource) which receives the action. - - ''' - - def __init__(self, action=None, network=None, target=None): - self.action = action - self.network = network - self.target = target - - def validate(self): - if not(self.action and self.network): - raise exceptions.ValidationError('Social interactions need to have at least the "network" and "action" attributes defined.') - - -class Transaction(object): - ''' - Represents parameters for a Transaction call - - Properties: - order_id -- Order ID, will be mapped to "utmtid" parameter - affiliation -- Affiliation, Will be mapped to "utmtst" parameter - total -- Total Cost, will be mapped to "utmtto" parameter - tax -- Tax Cost, will be mapped to "utmttx" parameter - shipping -- Shipping Cost, values as for unit and price, will be mapped to "utmtsp" parameter - city -- Billing City, will be mapped to "utmtci" parameter - state -- Billing Region, will be mapped to "utmtrg" parameter - country -- Billing Country, will be mapped to "utmtco" parameter - items -- @entity.Items in a transaction - - ''' - def __init__(self): - self.items = [] - self.order_id = None - self.affiliation = None - self.total = None - self.tax = None - self.shipping = None - self.city = None - self.state = None - self.country = None - - def __setattr__(self, name, value): - if name == 'order_id': - for itm in self.items: - itm.order_id = value - object.__setattr__(self, name, value) - - def validate(self): - if len(self.items) == 0: - raise exceptions.ValidationError('Transaction need to consist of at least one item') - - def add_item(self, item): - ''' item of type entities.Item ''' - if isinstance(item, Item): - item.order_id = self.order_id - self.items.append(item) - - -class Visitor(object): - ''' - You should serialize this object and store it in the user database to keep it - persistent for the same user permanently (similar to the "__umta" cookie of - the GA Javascript client). - - Properties: - unique_id -- Unique user ID, will be part of the "__utma" cookie parameter - first_visit_time -- Time of the very first visit of this user, will be part of the "__utma" cookie parameter - previous_visit_time -- Time of the previous visit of this user, will be part of the "__utma" cookie parameter - current_visit_time -- Time of the current visit of this user, will be part of the "__utma" cookie parameter - visit_count -- Amount of total visits by this user, will be part of the "__utma" cookie parameter - ip_address -- IP Address of the end user, will be mapped to "utmip" parameter and "X-Forwarded-For" request header - user_agent -- User agent string of the end user, will be mapped to "User-Agent" request header - locale -- Locale string (country part optional) will be mapped to "utmul" parameter - flash_version -- Visitor's Flash version, will be maped to "utmfl" parameter - java_enabled -- Visitor's Java support, will be mapped to "utmje" parameter - screen_colour_depth -- Visitor's screen color depth, will be mapped to "utmsc" parameter - screen_resolution -- Visitor's screen resolution, will be mapped to "utmsr" parameter - ''' - def __init__(self): - now = datetime.utcnow() - - self.unique_id = None - self.first_visit_time = now - self.previous_visit_time = now - self.current_visit_time = now - self.visit_count = 1 - self.ip_address = None - self.user_agent = None - self.locale = None - self.flash_version = None - self.java_enabled = None - self.screen_colour_depth = None - self.screen_resolution = None - - def __setattr__(self, name, value): - if name == 'unique_id': - if value and (value < 0 or value > 0x7fffffff): - raise ValueError('Visitor unique ID has to be a 32-bit integer between 0 and 0x7fffffff') - object.__setattr__(self, name, value) - - def __getattribute__(self, name): - if name == 'unique_id': - tmp = object.__getattribute__(self, name) - if tmp is None: - self.unique_id = self.generate_unique_id() - return object.__getattribute__(self, name) - - def __getstate__(self): - state = self.__dict__ - if state.get('user_agent') is None: - state['unique_id'] = self.generate_unique_id() - - return state - - def extract_from_utma(self, utma): - ''' - Will extract information for the "unique_id", "first_visit_time", "previous_visit_time", - "current_visit_time" and "visit_count" properties from the given "__utma" cookie value. - ''' - parts = utma.split('.') - if len(parts) != 6: - raise ValueError('The given "__utma" cookie value is invalid.') - - self.unique_id = int(parts[1]) - self.first_visit_time = utils.convert_ga_timestamp(parts[2]) - self.previous_visit_time = utils.convert_ga_timestamp(parts[3]) - self.current_visit_time = utils.convert_ga_timestamp(parts[4]) - self.visit_count = int(parts[5]) - - return self - - def extract_from_server_meta(self, meta): - ''' - Will extract information for the "ip_address", "user_agent" and "locale" - properties from the given WSGI REQUEST META variable or equivalent. - ''' - if 'REMOTE_ADDR' in meta and meta['REMOTE_ADDR']: - ip = None - for key in ('HTTP_X_FORWARDED_FOR', 'REMOTE_ADDR'): - if key in meta and not ip: - ips = meta.get(key, '').split(',') - ip = ips[-1].strip() - if not utils.is_valid_ip(ip): - ip = '' - if utils.is_private_ip(ip): - ip = '' - if ip: - self.ip_address = ip - - if 'HTTP_USER_AGENT' in meta and meta['HTTP_USER_AGENT']: - self.user_agent = meta['HTTP_USER_AGENT'] - - if 'HTTP_ACCEPT_LANGUAGE' in meta and meta['HTTP_ACCEPT_LANGUAGE']: - user_locals = [] - matched_locales = utils.validate_locale(meta['HTTP_ACCEPT_LANGUAGE']) - if matched_locales: - lang_lst = map((lambda x: x.replace('-', '_')), (i[1] for i in matched_locales)) - quality_lst = map((lambda x: x and x or 1), (float(i[4] and i[4] or '0') for i in matched_locales)) - lang_quality_map = map((lambda x, y: (x, y)), lang_lst, quality_lst) - user_locals = [x[0] for x in sorted(lang_quality_map, key=itemgetter(1), reverse=True)] - - if user_locals: - self.locale = user_locals[0] - - return self - - def generate_hash(self): - '''Generates a hashed value from user-specific properties.''' - tmpstr = "%s%s%s" % (self.user_agent, self.screen_resolution, self.screen_colour_depth) - return utils.generate_hash(tmpstr) - - def generate_unique_id(self): - '''Generates a unique user ID from the current user-specific properties.''' - return ((utils.get_32bit_random_num() ^ self.generate_hash()) & 0x7fffffff) - - def add_session(self, session): - ''' - Updates the "previousVisitTime", "currentVisitTime" and "visitCount" - fields based on the given session object. - ''' - start_time = session.start_time - if start_time != self.current_visit_time: - self.previous_visit_time = self.current_visit_time - self.current_visit_time = start_time - self.visit_count = self.visit_count + 1 diff --git a/libs/pyga/exceptions.py b/libs/pyga/exceptions.py deleted file mode 100644 index 15e676c5d..000000000 --- a/libs/pyga/exceptions.py +++ /dev/null @@ -1,2 +0,0 @@ -class ValidationError(Exception): - pass diff --git a/libs/pyga/requests.py b/libs/pyga/requests.py deleted file mode 100644 index 4a5f47c5c..000000000 --- a/libs/pyga/requests.py +++ /dev/null @@ -1,1047 +0,0 @@ -# -*- coding: utf-8 -*- - -import logging -import calendar -from math import floor -from pyga.entities import Campaign, CustomVariable, Event, Item, Page, Session, SocialInteraction, Transaction, Visitor -import pyga.utils as utils -from six import itervalues -try: - from urllib import urlencode - from urllib2 import Request as urllib_request - from urllib2 import urlopen -except ImportError as e: - from urllib.parse import urlencode - from urllib.request import Request as urllib_request - from urllib.request import urlopen - -__author__ = "Arun KR (kra3) 2036 - - if not use_post: - url = '%s?%s' % (self.config.endpoint, query_string) - post = None - else: - url = self.config.endpoint - post = query_string - - headers = {} - headers['Host'] = self.config.endpoint.split('/')[2] - headers['User-Agent'] = self.user_agent or '' - headers['X-Forwarded-For'] = self.x_forwarded_for and self.x_forwarded_for or '' - - if use_post: - # Don't ask me why "text/plain", but ga.js says so :) - headers['Content-Type'] = 'text/plain' - headers['Content-Length'] = len(query_string) - - logger.debug(url) - if post: - logger.debug(post) - return urllib_request(url, post, headers) - - def build_parameters(self): - '''Marker implementation''' - return Parameters() - - def __send(self): - request = self.build_http_request() - response = None - - # Do not actually send the request if endpoint host is set to null - if self.config.endpoint: - response = urlopen( - request, timeout=self.config.request_timeout) - - return response - - def fire(self): - ''' - Simply delegates to send() if config option "queue_requests" is disabled - else enqueues the request into Q object: you should call pyga.shutdowon - as last statement, to actually send out all queued requests. - ''' - if self.config.queue_requests: - # Queuing results. You should call pyga.shutdown as last statement to send out requests. - self.__Q.add_wrapped_request((lambda: self.__send())) - else: - self.__send() - - -class Request(GIFRequest): - TYPE_PAGE = None - TYPE_EVENT = 'event' - TYPE_TRANSACTION = 'tran' - TYPE_ITEM = 'item' - TYPE_SOCIAL = 'social' - - ''' - This type of request is deprecated in favor of encoding custom variables - within the "utme" parameter, but we include it here for completeness - ''' - TYPE_CUSTOMVARIABLE = 'var' - - X10_CUSTOMVAR_NAME_PROJECT_ID = 8 - X10_CUSTOMVAR_VALUE_PROJCT_ID = 9 - X10_CUSTOMVAR_SCOPE_PROJECT_ID = 11 - - def __init__(self, config, tracker, visitor, session): - super(Request, self).__init__(config) - self.tracker = tracker - self.visitor = visitor - self.session = session - - def build_http_request(self): - self.x_forwarded_for = self.visitor.ip_address - self.user_agent = self.visitor.user_agent - - # Increment session track counter for each request - self.session.track_count = self.session.track_count + 1 - - #http://code.google.com/intl/de-DE/apis/analytics/docs/tracking/eventTrackerGuide.html#implementationConsiderations - if self.session.track_count > 500: - logger.warning('Google Analytics does not guarantee to process more than 500 requests per session.') - - if self.tracker.campaign: - self.tracker.campaign.response_count = self.tracker.campaign.response_count + 1 - - return super(Request, self).build_http_request() - - def build_parameters(self): - params = Parameters() - params.utmac = self.tracker.account_id - params.utmhn = self.tracker.domain_name - params.utmt = self.get_type() - params.utmn = utils.get_32bit_random_num() - ''' - The "utmip" parameter is only relevant if a mobile analytics ID - (MO-XXXXXX-X) was given - ''' - params.utmip = self.visitor.ip_address - params.aip = self.tracker.config.anonimize_ip_address and 1 or None - - # Add override User-Agent parameter (&ua) and override IP address - # parameter (&uip). Note that the override IP address parameter is - # always anonymized, as if &aip were present (see - # https://developers.google.com/analytics/devguides/collection/protocol/v1/parameters#uip) - params.ua = self.visitor.user_agent - params.uip = utils.anonymize_ip(self.visitor.ip_address) - - if params.aip: - # If anonimization of ip enabled? then! - params.utmip = utils.anonymize_ip(params.utmip) - - params.utmhid = self.session.session_id - params.utms = self.session.track_count - params = self.build_visitor_parameters(params) - params = self.build_custom_variable_parameters(params) - params = self.build_campaign_parameters(params) - params = self.build_cookie_parameters(params) - return params - - def build_visitor_parameters(self, params): - if self.visitor.locale: - params.utmul = self.visitor.locale.replace('_', '-').lower() - - if self.visitor.flash_version: - params.utmfl = self.visitor.flash_version - - if self.visitor.java_enabled: - params.utje = self.visitor.java_enabled - - if self.visitor.screen_colour_depth: - params.utmsc = '%s-bit' % (self.visitor.screen_colour_depth) - - if self.visitor.screen_resolution: - params.utmsr = self.visitor.screen_resolution - - return params - - def build_custom_variable_parameters(self, params): - custom_vars = self.tracker.custom_variables - - if custom_vars: - if len(custom_vars) > 5: - logger.warning('The sum of all custom variables cannot exceed 5 in any given request.') - - x10 = X10() - x10.clear_key(self.X10_CUSTOMVAR_NAME_PROJECT_ID) - x10.clear_key(self.X10_CUSTOMVAR_VALUE_PROJCT_ID) - x10.clear_key(self.X10_CUSTOMVAR_SCOPE_PROJECT_ID) - - for cvar in itervalues(custom_vars): - name = utils.encode_uri_components(cvar.name) - value = utils.encode_uri_components(cvar.value) - x10.set_key( - self.X10_CUSTOMVAR_NAME_PROJECT_ID, cvar.index, name) - x10.set_key( - self.X10_CUSTOMVAR_VALUE_PROJCT_ID, cvar.index, value) - - if cvar.scope and cvar.scope != CustomVariable.SCOPE_PAGE: - x10.set_key(self.X10_CUSTOMVAR_SCOPE_PROJECT_ID, - cvar.index, cvar.scope) - - params.utme = '%s%s' % (params.utme, x10.render_url_string()) - - return params - - def build_campaign_parameters(self, params): - campaign = self.tracker.campaign - if campaign: - params._utmz = '%s.%s.%s.%s.' % ( - self._generate_domain_hash(), - calendar.timegm(campaign.creation_time.timetuple()), - self.visitor.visit_count, - campaign.response_count, - ) - - param_map = { - 'utmcid': campaign.id, - 'utmcsr': campaign.source, - 'utmgclid': campaign.g_click_id, - 'utmdclid': campaign.d_click_id, - 'utmccn': campaign.name, - 'utmcmd': campaign.medium, - 'utmctr': campaign.term, - 'utmcct': campaign.content, - } - - for k, v in param_map.items(): - if v: - # Only spaces and pluses get escaped in gaforflash and ga.js, so we do the same - params._utmz = '%s%s=%s%s' % (params._utmz, k, - v.replace('+', '%20').replace(' ', '%20'), - Campaign.CAMPAIGN_DELIMITER - ) - - params._utmz = params._utmz.rstrip(Campaign.CAMPAIGN_DELIMITER) - - return params - - def build_cookie_parameters(self, params): - domain_hash = self._generate_domain_hash() - params._utma = "%s.%s.%s.%s.%s.%s" % ( - domain_hash, - self.visitor.unique_id, - calendar.timegm(self.visitor.first_visit_time.timetuple()), - calendar.timegm(self.visitor.previous_visit_time.timetuple()), - calendar.timegm(self.visitor.current_visit_time.timetuple()), - self.visitor.visit_count - ) - params._utmb = '%s.%s.10.%s' % ( - domain_hash, - self.session.track_count, - calendar.timegm(self.session.start_time.timetuple()), - ) - params._utmc = domain_hash - cookies = [] - cookies.append('__utma=%s;' % params._utma) - if params._utmz: - cookies.append('__utmz=%s;' % params._utmz) - if params._utmv: - cookies.append('__utmv=%s;' % params._utmv) - - params.utmcc = '+'.join(cookies) - return params - - def _generate_domain_hash(self): - hash_val = 1 - if self.tracker.allow_hash: - hash_val = utils.generate_hash(self.tracker.domain_name) - - return hash_val - - -class ItemRequest(Request): - def __init__(self, config, tracker, visitor, session, item): - super(ItemRequest, self).__init__(config, tracker, visitor, session) - self.item = item - - def get_type(self): - return ItemRequest.TYPE_ITEM - - def build_parameters(self): - params = super(ItemRequest, self).build_parameters() - params.utmtid = self.item.order_id - params.utmipc = self.item.sku - params.utmipn = self.item.name - params.utmiva = self.item.variation - params.utmipr = self.item.price - params.utmiqt = self.item.quantity - return params - - def build_visitor_parameters(self, parameters): - ''' - The GA Javascript client doesn't send any visitor information for - e-commerce requests, so we don't either. - ''' - return parameters - - def build_custom_variable_parameters(self, parameters): - ''' - The GA Javascript client doesn't send any custom variables for - e-commerce requests, so we don't either. - ''' - return parameters - - -class PageViewRequest(Request): - X10_SITESPEED_PROJECT_ID = 14 - - def __init__(self, config, tracker, visitor, session, page): - super( - PageViewRequest, self).__init__(config, tracker, visitor, session) - self.page = page - - def get_type(self): - return PageViewRequest.TYPE_PAGE - - def build_parameters(self): - params = super(PageViewRequest, self).build_parameters() - params.utmp = self.page.path - params.utmdt = self.page.title - - if self.page.charset: - params.utmcs = self.page.charset - - if self.page.referrer: - params.utmr = self.page.referrer - - if self.page.load_time: - if params.utmn % 100 < self.config.site_speed_sample_rate: - x10 = X10() - x10.clear_key(self.X10_SITESPEED_PROJECT_ID) - x10.clear_value(self.X10_SITESPEED_PROJECT_ID) - - # from ga.js - key = max(min(floor(self.page.load_time / 100), 5000), 0) * 100 - x10.set_key( - self.X10_SITESPEED_PROJECT_ID, X10.OBJECT_KEY_NUM, key) - x10.set_value(self.X10_SITESPEED_PROJECT_ID, - X10.VALUE_VALUE_NUM, self.page.load_time) - params.utme = '%s%s' % (params.utme, x10.render_url_string()) - - return params - - -class EventRequest(Request): - X10_EVENT_PROJECT_ID = 5 - - def __init__(self, config, tracker, visitor, session, event): - super(EventRequest, self).__init__(config, tracker, visitor, session) - self.event = event - - def get_type(self): - return EventRequest.TYPE_EVENT - - def build_parameters(self): - params = super(EventRequest, self).build_parameters() - x10 = X10() - x10.clear_key(self.X10_EVENT_PROJECT_ID) - x10.clear_value(self.X10_EVENT_PROJECT_ID) - x10.set_key(self.X10_EVENT_PROJECT_ID, X10.OBJECT_KEY_NUM, - self.event.category) - x10.set_key( - self.X10_EVENT_PROJECT_ID, X10.TYPE_KEY_NUM, self.event.action) - - if self.event.label: - x10.set_key(self.X10_EVENT_PROJECT_ID, - X10.LABEL_KEY_NUM, self.event.label) - - if self.event.value: - x10.set_value(self.X10_EVENT_PROJECT_ID, - X10.VALUE_VALUE_NUM, self.event.value) - - params.utme = "%s%s" % (params.utme, x10.render_url_string()) - - if self.event.noninteraction: - params.utmni = 1 - - return params - - -class SocialInteractionRequest(Request): - def __init__(self, config, tracker, visitor, session, social_interaction, page): - super(SocialInteractionRequest, self).__init__(config, - tracker, visitor, session) - self.social_interaction = social_interaction - self.page = page - - def get_type(self): - return SocialInteractionRequest.TYPE_SOCIAL - - def build_parameters(self): - params = super(SocialInteractionRequest, self).build_parameters() - - tmppagepath = self.social_interaction.target - if tmppagepath is None: - tmppagepath = self.page.path - - params.utmsn = self.social_interaction.network - params.utmsa = self.social_interaction.action - params.utmsid = tmppagepath - return params - - -class TransactionRequest(Request): - def __init__(self, config, tracker, visitor, session, transaction): - super(TransactionRequest, self).__init__(config, tracker, - visitor, session) - self.transaction = transaction - - def get_type(self): - return TransactionRequest.TYPE_TRANSACTION - - def build_parameters(self): - params = super(TransactionRequest, self).build_parameters() - params.utmtid = self.transaction.order_id - params.utmtst = self.transaction.affiliation - params.utmtto = self.transaction.total - params.utmttx = self.transaction.tax - params.utmtsp = self.transaction.shipping - params.utmtci = self.transaction.city - params.utmtrg = self.transaction.state - params.utmtco = self.transaction.country - return params - - def build_visitor_parameters(self, parameters): - ''' - The GA Javascript client doesn't send any visitor information for - e-commerce requests, so we don't either. - ''' - return parameters - - def build_custom_variable_parameters(self, parameters): - ''' - The GA Javascript client doesn't send any custom variables for - e-commerce requests, so we don't either. - ''' - return parameters - - -class Config(object): - ''' - Configurations for Google Analytics: Server Side - - Properties: - error_severity -- How strict should errors get handled? After all, - we do just do some tracking stuff here, and errors shouldn't - break an application's functionality in production. - RECOMMENDATION: Exceptions during deveopment, warnings in production. - queue_requests -- Whether to just queue all requests on HttpRequest.fire() - and actually send them on shutdown after all other tasks are done. - This has two advantages: - 1) It effectively doesn't affect app performance - 2) It can e.g. handle custom variables that were set after scheduling a request - fire_and_forget -- Whether to make asynchronous requests to GA without - waiting for any response (speeds up doing requests). - logging_callback -- Logging callback, registered via setLoggingCallback(). - Will be fired whenever a request gets sent out and receives the - full HTTP request as the first and the full HTTP response - (or null if the "fireAndForget" option or simulation mode are used) as the 2nd argument. - request_timeout -- Seconds (float allowed) to wait until timeout when - connecting to the Google analytics endpoint host. - endpoint -- Google Analytics tracking request endpoint. Can be set to null to - silently simulate (and log) requests without actually sending them. - anonimize_ip_address -- Whether to anonymize IP addresses within Google Analytics - by stripping the last IP address block, will be mapped to "aip" parameter. - site_speed_sample_rate -- Defines a new sample set size (0-100) for - Site Speed data collection. By default, a fixed 1% sampling of your site - visitors make up the data pool from which the Site Speed metrics are derived. - - ''' - ERROR_SEVERITY_SILECE = 0 - ERROR_SEVERITY_PRINT = 1 - ERROR_SEVERITY_RAISE = 2 - - def __init__(self): - self.error_severity = Config.ERROR_SEVERITY_RAISE - self.queue_requests = False - # self.fire_and_forget = False # not supported as of now - # self.logging_callback = False # not supported as of now - self.request_timeout = 1 - self.endpoint = 'http://www.google-analytics.com/__utm.gif' - self.anonimize_ip_address = False - self.site_speed_sample_rate = 1 - - def __setattr__(self, name, value): - if name == 'site_speed_sample_rate': - if value and (value < 0 or value > 100): - raise ValueError('For consistency with ga.js, sample rates must be specified as a number between 0 and 100.') - object.__setattr__(self, name, value) - - -class Parameters(object): - ''' - This simple class is mainly meant to be a well-documented overview - of all possible GA tracking parameters. - - http://code.google.com/apis/analytics/docs/tracking/gaTrackingTroubleshooting.html#gifParameters - - General Parameters: - utmwv -- Google Analytics client version - utmac -- Google Analytics account ID - utmhn -- Host Name - utmt -- Indicates the type of request, which is one of null (for page), - "event", "tran", "item", "social", "var" (deprecated) or "error" - (used by ga.js for internal client error logging). - utms -- Contains the amount of requests done in this session. Added in ga.js v4.9.2. - utmn -- Unique ID (random number) generated for each GIF request - utmcc -- Contains all cookie values, see below - utme -- Extensible Parameter, used for events and custom variables - utmni -- Event "non-interaction" parameter. By default, the event hit will impact a visitor's bounce rate. - By setting this parameter to 1, this event hit will not be used in bounce rate calculations. - aip -- Whether to anonymize IP addresses within Google Analytics by stripping the last IP address block, either null or 1 - utmu -- Used for GA-internal statistical client function usage and error tracking, - not implemented in php-ga as of now, but here for documentation completeness. - http://glucik.blogspot.com/2011/02/utmu-google-analytics-request-parameter.html - - Page Parameters: - utmp -- Page request URI - utmdt -- Page title - utmcs -- Charset encoding (default "-") - utmr -- Referer URL (default "-" or "0" for internal purposes) - - Visitor Parameters: - utmip -- IP Address of the end user, found in GA for Mobile examples, but sadly seems to be ignored in normal GA use - utmul -- Visitor's locale string (all lower-case, country part optional) - utmfl -- Visitor's Flash version (default "-") - utmje -- Visitor's Java support, either 0 or 1 (default "-") - utmsc -- Visitor's screen color depth - utmsr -- Visitor's screen resolution - _utma -- Visitor tracking cookie parameter. - - Session Parameters: - utmhid -- Hit id for revenue per page tracking for AdSense, a random per-session ID - _utmb -- Session timeout cookie parameter. - _utmc -- Session tracking cookie parameter. - utmipc -- Product Code. This is the sku code for a given product. - utmipn -- Product Name - utmipr -- Unit Price. Value is set to numbers only. - utmiqt -- Unit Quantity. - utmiva -- Variations on an item. - utmtid -- Order ID. - utmtst -- Affiliation - utmtto -- Total Cost - utmttx -- Tax Cost - utmtsp -- Shipping Cost - utmtci -- Billing City - utmtrg -- Billing Region - utmtco -- Billing Country - - Campaign Parameters: - utmcn -- Starts a new campaign session. Either utmcn or utmcr is present on any given request, - but never both at the same time. Changes the campaign tracking data; - but does not start a new session. Either 1 or not set. - Found in gaforflash but not in ga.js, so we do not use it, - but it will stay here for documentation completeness. - utmcr -- Indicates a repeat campaign visit. This is set when any subsequent clicks occur on the - same link. Either utmcn or utmcr is present on any given request, - but never both at the same time. Either 1 or not set. - Found in gaforflash but not in ga.js, so we do not use it, - but it will stay here for documentation completeness. - utmcid -- Campaign ID, a.k.a. "utm_id" query parameter for ga.js - utmcsr -- Source, a.k.a. "utm_source" query parameter for ga.js - utmgclid -- Google AdWords Click ID, a.k.a. "gclid" query parameter for ga.js - utmdclid -- Not known for sure, but expected to be a DoubleClick Ad Click ID. - utmccn -- Name, a.k.a. "utm_campaign" query parameter for ga.js - utmcmd -- Medium, a.k.a. "utm_medium" query parameter for ga.js - utmctr -- Terms/Keywords, a.k.a. "utm_term" query parameter for ga.js - utmcct -- Ad Content Description, a.k.a. "utm_content" query parameter for ga.js - utmcvr -- Unknown so far. Found in ga.js. - _utmz -- Campaign tracking cookie parameter. - - Social Tracking Parameters: - utmsn -- The network on which the action occurs - utmsa -- The type of action that happens - utmsid -- The page URL from which the action occurred. - - Google Website Optimizer (GWO) parameters: - _utmx -- Website Optimizer cookie parameter. - - Custom Variables parameters (deprecated): - _utmv -- Deprecated custom variables cookie parameter. - - ''' - - def __init__(self): - # General Parameters - self.utmwv = Tracker.VERSION - self.utmac = '' - self.utmhn = '' - self.utmt = '' - self.utms = '' - self.utmn = '' - self.utmcc = '' - self.utme = '' - self.utmni = '' - self.aip = '' - self.utmu = '' - - # Page Parameters - self.utmp = '' - self.utmdt = '' - self.utmcs = '-' - self.utmr = '-' - - # Visitor Parameters - self.utmip = '' - self.utmul = '' - self.utmfl = '-' - self.utmje = '-' - self.utmsc = '' - self.utmsr = '' - ''' - Visitor tracking cookie __utma - - This cookie is typically written to the browser upon the first - visit to your site from that web browser. If the cookie has been - deleted by the browser operator, and the browser subsequently - visits your site, a new __utma cookie is written with a different unique ID. - - This cookie is used to determine unique visitors to your site and - it is updated with each page view. Additionally, this cookie is - provided with a unique ID that Google Analytics uses to ensure both the - validity and accessibility of the cookie as an extra security measure. - - Expiration: 2 years from set/update. - Format: __utma=..... - ''' - self._utma = '' - - # Session Parameters - self.utmhid = '' - ''' - Session timeout cookie parameter __utmb - - Will never be sent with requests, but stays here for documentation completeness. - - This cookie is used to establish and continue a user session with your site. - When a user views a page on your site, the Google Analytics code attempts to update this cookie. - If it does not find the cookie, a new one is written and a new session is established. - - Each time a user visits a different page on your site, this cookie is updated to expire in 30 minutes, - thus continuing a single session for as long as user activity continues within 30-minute intervals. - - This cookie expires when a user pauses on a page on your site for longer than 30 minutes. - You can modify the default length of a user session with the setSessionTimeout() method. - - Expiration: 30 minutes from set/update. - - Format: __utmb=... - - ''' - self._utmb = '' - ''' - Session tracking cookie parameter __utmc - - Will never be sent with requests, but stays here for documentation completeness. - - This cookie operates in conjunction with the __utmb cookie to - determine whether or not to establish a new session for the user. - In particular, this cookie is not provided with an expiration date, - so it expires when the user exits the browser. - - Should a user visit your site, exit the browser and then return to your website within 30 minutes, - the absence of the __utmc cookie indicates that a new session needs to be established, - despite the fact that the __utmb cookie has not yet expired. - - Expiration: Not set. - - Format: __utmc= - - ''' - self._utmc = '' - self.utmipc = '' - self.utmipn = '' - self.utmipr = '' - self.utmiqt = '' - self.utmiva = '' - self.utmtid = '' - self.utmtst = '' - self.utmtto = '' - self.utmttx = '' - self.utmtsp = '' - self.utmtci = '' - self.utmtrg = '' - self.utmtco = '' - - # Campaign Parameters - self.utmcn = '' - self.utmcr = '' - self.utmcid = '' - self.utmcsr = '' - self.utmgclid = '' - self.utmdclid = '' - self.utmccn = '' - self.utmcmd = '' - self.utmctr = '' - self.utmcct = '' - self.utmcvr = '' - ''' - Campaign tracking cookie parameter. - - This cookie stores the type of referral used by the visitor to reach your site, - whether via a direct method, a referring link, a website search, or a campaign such as an ad or an email link. - - It is used to calculate search engine traffic, ad campaigns and page navigation within your own site. - The cookie is updated with each page view to your site. - - Expiration: 6 months from set/update. - - Format: __utmz=.... - - ''' - self._utmz = '' - - # Social Tracking Parameters - self.utmsn = '' - self.utmsa = '' - self.utmsid = '' - - # Google Website Optimizer (GWO) parameters - ''' - Website Optimizer cookie parameter. - - This cookie is used by Website Optimizer and only set when Website - Optimizer is used in combination with GA. - See the Google Website Optimizer Help Center for details. - - Expiration: 2 years from set/update. - ''' - self._utmx = '' - - # Custom Variables parameters (deprecated) - ''' - Deprecated custom variables cookie parameter. - - This cookie parameter is no longer relevant as of migration from setVar() to - setCustomVar() and hence not supported by this library, - but will stay here for documentation completeness. - - The __utmv cookie passes the information provided via the setVar() method, - which you use to create a custom user segment. - - Expiration: 2 years from set/update. - - Format: __utmv=. - - ''' - self._utmv = '' - - def get_parameters(self): - ''' - Get all gif request parameters out of the class in a dict form. - Attributes starting with _ are cookie names, so we dont need them. - ''' - params = {} - attribs = vars(self) - for attr in attribs: - if attr[0] != '_': - val = getattr(self, attr) - if val: - params[attr] = val - - return params - - -class Tracker(object): - ''' - Act like a Manager of all files - - Properties: - account_id -- Google Analytics account ID, will be mapped to "utmac" parameter - domain_name -- Host Name, will be mapped to "utmhn" parameter - allow_hash -- Whether to generate a unique domain hash, - default is true to be consistent with the GA Javascript Client - custom_variables -- CustomVariable instances - campaign -- Campaign instance - ''' - - ''' - Google Analytics client version on which this library is built upon, - will be mapped to "utmwv" parameter. - - This doesn't necessarily mean that all features of the corresponding - ga.js version are implemented but rather that the requests comply - with these of ga.js. - - http://code.google.com/apis/analytics/docs/gaJS/changelog.html - ''' - VERSION = '5.3.0' - config = Config() - - def __init__(self, account_id='', domain_name='', conf=None): - self.account_id = account_id - self.domain_name = domain_name - self.allow_hash = True - self.custom_variables = {} - self.campaign = None - if isinstance(conf, Config): - Tracker.config = conf - - def __setattr__(self, name, value): - if name == 'account_id': - if value and not utils.is_valid_google_account(value): - raise ValueError( - 'Given Google Analytics account ID is not valid') - - elif name == 'campaign': - if isinstance(value, Campaign): - value.validate() - else: - value = None - - object.__setattr__(self, name, value) - - def add_custom_variable(self, custom_var): - ''' - Equivalent of _setCustomVar() in GA Javascript client - http://code.google.com/apis/analytics/docs/tracking/gaTrackingCustomVariables.html - ''' - if not isinstance(custom_var, CustomVariable): - return - - custom_var.validate() - index = custom_var.index - self.custom_variables[index] = custom_var - - def remove_custom_variable(self, index): - '''Equivalent of _deleteCustomVar() in GA Javascript client.''' - if index in self.custom_variables: - del self.custom_variables[index] - - def track_pageview(self, page, session, visitor): - '''Equivalent of _trackPageview() in GA Javascript client.''' - params = { - 'config': self.config, - 'tracker': self, - 'visitor': visitor, - 'session': session, - 'page': page, - } - request = PageViewRequest(**params) - request.fire() - - def track_event(self, event, session, visitor): - '''Equivalent of _trackEvent() in GA Javascript client.''' - event.validate() - - params = { - 'config': self.config, - 'tracker': self, - 'visitor': visitor, - 'session': session, - 'event': event, - } - request = EventRequest(**params) - request.fire() - - def track_transaction(self, transaction, session, visitor): - '''Combines _addTrans(), _addItem() (indirectly) and _trackTrans() of GA Javascript client.''' - transaction.validate() - - params = { - 'config': self.config, - 'tracker': self, - 'visitor': visitor, - 'session': session, - 'transaction': transaction, - } - request = TransactionRequest(**params) - request.fire() - - for item in transaction.items: - item.validate() - - params = { - 'config': self.config, - 'tracker': self, - 'visitor': visitor, - 'session': session, - 'item': item, - } - request = ItemRequest(**params) - request.fire() - - def track_social(self, social_interaction, page, session, visitor): - '''Equivalent of _trackSocial() in GA Javascript client.''' - params = { - 'config': self.config, - 'tracker': self, - 'visitor': visitor, - 'session': session, - 'social_interaction': social_interaction, - 'page': page, - } - request = SocialInteractionRequest(**params) - request.fire() - - -class X10(object): - __KEY = 'k' - __VALUE = 'v' - __DELIM_BEGIN = '(' - __DELIM_END = ')' - __DELIM_SET = '*' - __DELIM_NUM_VALUE = '!' - __ESCAPE_CHAR_MAP = { - "'": "'0", - ')': "'1", - '*': "'2", - '!': "'3", - } - __MINIMUM = 1 - - OBJECT_KEY_NUM = 1 - TYPE_KEY_NUM = 2 - LABEL_KEY_NUM = 3 - VALUE_VALUE_NUM = 1 - - def __init__(self): - self.project_data = {} - - def has_project(self, project_id): - return project_id in self.project_data - - def set_key(self, project_id, num, value): - self.__set_internal(project_id, X10.__KEY, num, value) - - def get_key(self, project_id, num): - return self.__get_internal(project_id, X10.__KEY, num) - - def clear_key(self, project_id): - self.__clear_internal(project_id, X10.__KEY) - - def set_value(self, project_id, num, value): - self.__set_internal(project_id, X10.__VALUE, num, value) - - def get_value(self, project_id, num): - return self.__get_internal(project_id, X10.__VALUE, num) - - def clear_value(self, project_id): - self.__clear_internal(project_id, X10.__VALUE) - - def __set_internal(self, project_id, _type, num, value): - '''Shared internal implementation for setting an X10 data type.''' - if project_id not in self.project_data: - self.project_data[project_id] = {} - - if _type not in self.project_data[project_id]: - self.project_data[project_id][_type] = {} - - self.project_data[project_id][_type][num] = value - - def __get_internal(self, project_id, _type, num): - ''' Shared internal implementation for getting an X10 data type.''' - if num in self.project_data.get(project_id, {}).get(_type, {}): - return self.project_data[project_id][_type][num] - return None - - def __clear_internal(self, project_id, _type): - ''' - Shared internal implementation for clearing all X10 data - of a type from a certain project. - ''' - if project_id in self.project_data and _type in self.project_data[project_id]: - del self.project_data[project_id][_type] - - def __escape_extensible_value(self, value): - '''Escape X10 string values to remove ambiguity for special characters.''' - def _translate(char): - try: - return self.__ESCAPE_CHAR_MAP[char] - except KeyError: - return char - - return ''.join(map(_translate, str(value))) - - def __render_data_type(self, data): - '''Given a data array for a certain type, render its string encoding.''' - result = [] - last_indx = 0 - - for indx, entry in sorted(data.items()): - if entry: - tmpstr = '' - - # Check if we need to append the number. If the last number was - # outputted, or if this is the assumed minimum, then we don't. - if indx != X10.__MINIMUM and indx - 1 != last_indx: - tmpstr = '%s%s%s' % (tmpstr, indx, X10.__DELIM_NUM_VALUE) - - tmpstr = '%s%s' % ( - tmpstr, self.__escape_extensible_value(entry)) - result.append(tmpstr) - - last_indx = indx - - return "%s%s%s" % (X10.__DELIM_BEGIN, X10.__DELIM_SET.join(result), X10.__DELIM_END) - - def __render_project(self, project): - '''Given a project array, render its string encoding.''' - result = '' - need_type_qualifier = False - - for val in X10.__KEY, X10.__VALUE: - if val in project: - data = project[val] - if need_type_qualifier: - result = '%s%s' % (result, val) - - result = '%s%s' % (result, self.__render_data_type(data)) - need_type_qualifier = False - else: - need_type_qualifier = True - - return result - - def render_url_string(self): - result = '' - for project_id, project in self.project_data.items(): - result = '%s%s%s' % ( - result, project_id, self.__render_project(project)) - - return result diff --git a/libs/pyga/utils.py b/libs/pyga/utils.py deleted file mode 100644 index e21f25218..000000000 --- a/libs/pyga/utils.py +++ /dev/null @@ -1,125 +0,0 @@ -# -*- coding: utf-8 -*- -from __future__ import unicode_literals -from random import randint -import re -import sys -from datetime import datetime - -try: - from urllib import quote -except ImportError as e: - from urllib.parse import quote - -if sys.version_info < (3,): - text_type = unicode -else: - text_type = str - - -__author__ = "Arun KR (kra3) " -__license__ = "Simplified BSD" - -RE_IP = re.compile(r'^[\d+]{1,3}\.[\d+]{1,3}\.[\d+]{1,3}\.[\d+]{1,3}$', re.I) -RE_PRIV_IP = re.compile(r'^(?:127\.0\.0\.1|10\.|192\.168\.|172\.(?:1[6-9]|2[0-9]|3[0-1])\.)') -RE_LOCALE = re.compile(r'(^|\s*,\s*)([a-zA-Z]{1,8}(-[a-zA-Z]{1,8})*)\s*(;\s*q\s*=\s*(1(\.0{0,3})?|0(\.[0-9]{0,3})))?', re.I) -RE_GA_ACCOUNT_ID = re.compile(r'^(UA|MO)-[0-9]*-[0-9]*$') -RE_FIRST_THREE_OCTETS_OF_IP = re.compile(r'^((\d{1,3}\.){3})\d{1,3}$') - -def convert_ga_timestamp(timestamp_string): - timestamp = float(timestamp_string) - if timestamp > ((2 ** 31) - 1): - timestamp /= 1000 - return datetime.utcfromtimestamp(timestamp) - -def get_32bit_random_num(): - return randint(0, 0x7fffffff) - -def is_valid_ip(ip): - return True if RE_IP.match(str(ip)) else False - -def is_private_ip(ip): - return True if RE_PRIV_IP.match(str(ip)) else False - -def validate_locale(locale): - return RE_LOCALE.findall(str(locale)) - -def is_valid_google_account(account): - return True if RE_GA_ACCOUNT_ID.match(str(account)) else False - -def generate_hash(tmpstr): - hash_val = 1 - - if tmpstr: - hash_val = 0 - for ordinal in map(ord, tmpstr[::-1]): - hash_val = ((hash_val << 6) & 0xfffffff) + ordinal + (ordinal << 14) - left_most_7 = hash_val & 0xfe00000 - if left_most_7 != 0: - hash_val ^= left_most_7 >> 21 - - return hash_val - -def anonymize_ip(ip): - if ip: - match = RE_FIRST_THREE_OCTETS_OF_IP.findall(str(ip)) - if match: - return '%s%s' % (match[0][0], '0') - - return '' - -def encode_uri_components(value): - '''Mimics Javascript's encodeURIComponent() function for consistency with the GA Javascript client.''' - return convert_to_uri_component_encoding(quote(value)) - -def convert_to_uri_component_encoding(value): - return value.replace('%21', '!').replace('%2A', '*').replace('%27', "'").replace('%28', '(').replace('%29', ')') - -# Taken from expicient.com BJs repo. -def stringify(s, stype=None, fn=None): - ''' Converts elements of a complex data structure to strings - - The data structure can be a multi-tiered one - with tuples and lists etc - This method will loop through each and convert everything to string. - For example - it can be - - [[{'a1': {'a2': {'a3': ('a4', timedelta(0, 563)), 'a5': {'a6': datetime()}}}}]] - which will be converted to - - [[{'a1': {'a2': {'a3': ('a4', '0:09:23'), 'a5': {'a6': '2009-05-27 16:19:52.401500' }}}}]] - - @param stype: If only one type of data element needs to be converted to - string without affecting others, stype can be used. - In the earlier example, if it is called with stringify(s, stype=datetime.timedelta) - the result would be - [[{'a1': {'a2': {'a3': ('a4', '0:09:23'), 'a5': {'a6': datetime() }}}}]] - - Also, even though the name is stringify, any function can be run on it, based on - parameter fn. If fn is None, it will be stringified. - - ''' - - if type(s) in [list, set, dict, tuple]: - if isinstance(s, dict): - for k in s: - s[k] = stringify(s[k], stype, fn) - elif type(s) in [list, set]: - for i, k in enumerate(s): - s[i] = stringify(k, stype, fn) - else: #tuple - tmp = [] - for k in s: - tmp.append(stringify(k, stype, fn)) - s = tuple(tmp) - else: - if fn: - if not stype or (stype == type(s)): - return fn(s) - else: - # To do str(s). But, str() can fail on unicode. So, use .encode instead - if not stype or (stype == type(s)): - try: - return text_type(s) - #return s.encode('ascii', 'replace') - except AttributeError: - return str(s) - except UnicodeDecodeError: - return s.decode('ascii', 'replace') - return s diff --git a/libs/version.txt b/libs/version.txt index 62e337c0b..e1dd091f3 100644 --- a/libs/version.txt +++ b/libs/version.txt @@ -13,6 +13,7 @@ flask-cors==3.0.10 flask-restx==1.0.3 Flask-SocketIO==5.3.1 Flask==2.2.2 +ga4mp==2.0.4 guess_language-spirit==0.5.3 guessit==3.5.0 jsonschema==4.17.0 @@ -20,7 +21,6 @@ knowit==0.4.0 peewee==3.15.3 py-pretty==1 pycountry==22.3.5 -pyga==2.6.2 pyrsistent==0.19.2 pysubs2==1.4.4 python-engineio==4.3.4