{ "nbformat": 4, "nbformat_minor": 0, "metadata": { "colab": { "name": "strava-activity-analysis.ipynb", "provenance": [], "collapsed_sections": [] }, "kernelspec": { "name": "python3", "display_name": "Python 3" } }, "cells": [ { "cell_type": "markdown", "metadata": { "id": "SWea7UFXjgnG", "colab_type": "text" }, "source": [ "# Setup" ] }, { "cell_type": "code", "metadata": { "id": "C9mtdgnR49kK", "colab_type": "code", "cellView": "form", "colab": {} }, "source": [ "#@title Install stravalib\n", "!pip install stravalib\n" ], "execution_count": null, "outputs": [] }, { "cell_type": "code", "metadata": { "id": "ktonxl4a67Gx", "colab_type": "code", "cellView": "form", "colab": {} }, "source": [ "#@title Imports\n", "\n", "import warnings\n", "warnings.filterwarnings(\"ignore\")\n", "\n", "import pandas as pd\n", "import numpy as np\n", "import stravalib.client\n", "from matplotlib import pyplot as plt\n", "import matplotlib as mpl\n", "import seaborn as sns\n", "\n", "sns.set()" ], "execution_count": null, "outputs": [] }, { "cell_type": "code", "metadata": { "id": "JyRLwn_G-c0y", "colab_type": "code", "cellView": "form", "colab": {} }, "source": [ "#@title Unit testing\n", "import unittest\n", "\n", "def RunTest(test_case_class):\n", " suite = unittest.makeSuite(test_case_class)\n", " runner = unittest.runner.TextTestRunner(verbosity=1)\n", " runner.run(suite)" ], "execution_count": null, "outputs": [] }, { "cell_type": "code", "metadata": { "id": "kjOWdZ2c77D1", "colab_type": "code", "cellView": "form", "colab": {} }, "source": [ "#@title Strava layer\n", "\n", "class Strava(object):\n", "\n", " def __init__(self, client_id, client_secret):\n", " self.client = stravalib.client.Client()\n", " self.client_id = client_id\n", " self.client_secret = client_secret\n", " \n", " @property\n", " def authorization_url(self):\n", " return self.client.authorization_url(\n", " client_id=self.client_id,\n", " redirect_uri='http://localhost')\n", " \n", " def exchange_code_for_token(self, code):\n", " self.access_token = (\n", " self.client.exchange_code_for_token(self.client_id,\n", " self.client_secret,\n", " code))\n", " \n", " def get_raw_activities(self):\n", " return self.client.get_activities(limit=1000000)\n", " import pandas as pd\n", "\n", " def get_raw_activities_as_dataframe(self):\n", " return pd.DataFrame.from_records(\n", " map(lambda a: a.to_dict(),\n", " self.get_raw_activities()))\n" ], "execution_count": null, "outputs": [] }, { "cell_type": "markdown", "metadata": { "id": "mmPqaGXBjkRV", "colab_type": "text" }, "source": [ "# Extract" ] }, { "cell_type": "code", "metadata": { "id": "QZphZ-n8-b9k", "colab_type": "code", "cellView": "form", "colab": {} }, "source": [ "#@title Get Strava authorization code\n", "\n", "STRAVA_CLIENT_ID = \"\" #@param {type: \"string\"}\n", "STRAVA_CLIENT_SECRET = \"\" #@param {type: \"string\"}\n", "\n", "strava = Strava(client_id=STRAVA_CLIENT_ID,\n", " client_secret=STRAVA_CLIENT_SECRET)\n", "\n", "print(\"\"\"Go to\n", "\n", "%s\n", "\n", "Copy the code from the URL in the browser after having been\n", "redirected from the Strava authorization page.\n", "\"\"\" %\n", "strava.authorization_url)\n", "\n", "CODE = input()\n", "\n", "strava.exchange_code_for_token(CODE)" ], "execution_count": null, "outputs": [] }, { "cell_type": "code", "metadata": { "id": "-kws_0-97MUV", "colab_type": "code", "cellView": "form", "colab": {} }, "source": [ "#@title Download Strava activities\n", "_raw_activities = strava.get_raw_activities_as_dataframe()" ], "execution_count": null, "outputs": [] }, { "cell_type": "markdown", "metadata": { "id": "8oHBfAZZjo2S", "colab_type": "text" }, "source": [ "# Transform and load" ] }, { "cell_type": "markdown", "metadata": { "id": "sYKw3-44-w2F", "colab_type": "text" }, "source": [ "## Functions" ] }, { "cell_type": "code", "metadata": { "id": "nLKTPrEyCmZ4", "colab_type": "code", "cellView": "form", "colab": {} }, "source": [ "#@title activities_table\n", "\n", "def activities_table():\n", " \"\"\"\n", " column type description\n", " ----------------------------------------------------------------\n", " upload_id int64 identifies the activity\n", " start_date pandas.Timestamp start time of the activity\n", " name string name of the activity\n", " distance int64 distance in meters\n", " moving_time pandas.Timedelta time spent moving\n", " elapsed_time pandas.Timedelta duration of activity\n", " elevation_gain int64 elevation gain in meters\n", " \"\"\"\n", "\n", " def select(df):\n", " return df[[\n", " 'upload_id',\n", " 'start_date',\n", " 'name',\n", " 'type',\n", " 'distance',\n", " 'moving_time',\n", " 'elapsed_time',\n", " 'total_elevation_gain',\n", " ]]\n", "\n", " def rename(df):\n", " return df.rename({\n", " 'total_elevation_gain': 'elevation_gain',\n", " }, axis=1)\n", "\n", " def convert(df):\n", " cp = df.copy()\n", " cp['distance'] = (df['distance'] / 1000).round().astype(int)\n", " cp['elevation_gain'] = df['elevation_gain'].round().astype(int)\n", " cp['start_date'] = pd.to_datetime(df['start_date'])\n", " cp['moving_time'] = pd.to_timedelta(df['moving_time']).apply(lambda td: td.total_seconds() / 3600.)\n", " cp['elapsed_time'] = pd.to_timedelta(df['elapsed_time']).apply(lambda td: td.total_seconds() / 3600.)\n", " return cp\n", " \n", " def where(df):\n", " return df[df['type'] == 'Run']\n", "\n", " def sort(df):\n", " return df.sort_values('start_date', ascending=True)\n", "\n", " return lambda df: (df\n", " .pipe(select)\n", " .pipe(rename)\n", " .pipe(convert)\n", " .pipe(where)\n", " .pipe(sort)\n", " .reset_index(drop=True))" ], "execution_count": null, "outputs": [] }, { "cell_type": "code", "metadata": { "id": "N4imCZJd9ZBS", "colab_type": "code", "cellView": "form", "colab": {} }, "source": [ "#@title align_to_period\n", "\n", "def align_to_period(period_range : pd.PeriodIndex):\n", " \"\"\"Aligns a data frame to a particular period range.\"\"\"\n", " def op(df):\n", " cp = (df\n", " .set_index('period')\n", " .align(\n", " other=period_range.to_frame()[[]],\n", " join='right',\n", " axis=0)[0])\n", " cp.index.rename('period', inplace=True)\n", " return cp.reset_index()\n", " return op\n", "\n", "\n", "class align_to_period_test(unittest.TestCase):\n", "\n", " def test_fills_missing_periods_in_dataframe(self):\n", " # Arrange\n", " inp = pd.DataFrame.from_records(\n", " [\n", " (pd.Period('2019Q2'), 2),\n", " (pd.Period('2019Q4'), 4),\n", " ],\n", " columns=['period', 'value'])\n", " q1_q2_q3_q4 = pd.period_range(\n", " start=pd.Period('2019Q1'),\n", " end=pd.Period('2019Q4'),\n", " freq='Q',\n", " )\n", " expected = pd.DataFrame.from_records(\n", " [\n", " (pd.Period('2019Q1'), None),\n", " (pd.Period('2019Q2'), 2),\n", " (pd.Period('2019Q3'), None),\n", " (pd.Period('2019Q4'), 4),\n", " ],\n", " columns=['period', 'value'])\n", " # Act\n", " actual = inp.pipe(align_to_period(q1_q2_q3_q4))\n", " # Assert\n", " assert actual.equals(expected)\n", "\n", " def test_drops_periods_in_dataframe(self):\n", " # Arrange\n", " inp = pd.DataFrame.from_records(\n", " [\n", " (pd.Period('2019Q1'), 1),\n", " (pd.Period('2019Q2'), 2),\n", " (pd.Period('2019Q3'), 3),\n", " (pd.Period('2019Q4'), 4),\n", " ],\n", " columns=['period', 'value'])\n", " q2_q3 = pd.period_range(\n", " start=pd.Period('2019Q2'),\n", " end=pd.Period('2019Q3'),\n", " freq='Q',\n", " )\n", " expected = pd.DataFrame.from_records(\n", " [\n", " (pd.Period('2019Q2'), 2),\n", " (pd.Period('2019Q3'), 3),\n", " ],\n", " columns=['period', 'value'])\n", " # Act\n", " actual = inp.pipe(align_to_period(q2_q3))\n", " # Assert\n", " assert actual.equals(expected)\n", "\n", "RunTest(align_to_period_test)" ], "execution_count": null, "outputs": [] }, { "cell_type": "code", "metadata": { "id": "AuXVIZf7G4sG", "colab_type": "code", "cellView": "form", "colab": {} }, "source": [ "#@title densify\n", "\n", "def densify():\n", " def op(df):\n", " if len(df) == 0:\n", " return df\n", " return df.pipe(\n", " align_to_period(\n", " pd.period_range(\n", " start=df['period'].min(),\n", " end=df['period'].max(),\n", " freq=pd.PeriodIndex(df['period']).freq,\n", " name='period')))\n", " return op\n", "\n", "\n", "class densify_test(unittest.TestCase):\n", "\n", " def test_densify(self):\n", " # Arrange\n", " inp = pd.DataFrame.from_records(\n", " [\n", " (pd.Period('2019Q2'), 2),\n", " (pd.Period('2019Q4'), 4),\n", " ],\n", " columns=['period', 'value'])\n", " expected = pd.DataFrame.from_records(\n", " [\n", " (pd.Period('2019Q2'), 2),\n", " (pd.Period('2019Q3'), None),\n", " (pd.Period('2019Q4'), 4),\n", " ],\n", " columns=['period', 'value'])\n", " # Act\n", " actual = inp.pipe(densify())\n", " # Assert\n", " assert actual.equals(expected)\n", " \n", " def test_densify_empty(self):\n", " # Arrange\n", " expected = pd.DataFrame(columns=['period', 'value'])\n", " # Act\n", " actual = expected.pipe(densify())\n", " # Assert\n", " assert actual.equals(expected)\n", "\n", "RunTest(densify_test)\n" ], "execution_count": null, "outputs": [] }, { "cell_type": "code", "metadata": { "id": "-MZszBx2x5Mt", "colab_type": "code", "cellView": "form", "colab": {} }, "source": [ "#@title fillna\n", "\n", "def fillna():\n", " \"\"\"Fill all N/A values with the respective zero element.\"\"\"\n", " def operator(df):\n", " r = df.copy() \n", " r['moving_time'].fillna(0, inplace=True)\n", " r['elapsed_time'].fillna(0, inplace=True)\n", " r['distance'].fillna(0, inplace=True)\n", " r['elevation_gain'].fillna(0, inplace=True)\n", " return r\n", " return operator\n", "\n", "\n", "class fillna_test(unittest.TestCase):\n", "\n", " def test_fillna(self):\n", " # Arrange\n", " inp = pd.DataFrame.from_records(\n", " [\n", " (pd.Period('2019Q2'), 24200., 232., 2., 3.,),\n", " (pd.Period('2019Q3'), None, None, None, None,),\n", " (pd.Period('2019Q4'), 15999., 843., 1., 2.5,),\n", " ],\n", " columns=['period', 'distance', 'elevation_gain', 'moving_time',\n", " 'elapsed_time'])\n", " expected = pd.DataFrame.from_records(\n", " [\n", " (pd.Period('2019Q2'), 24200., 232., 2., 3.,),\n", " (pd.Period('2019Q3'), 0., 0., 0., 0.,),\n", " (pd.Period('2019Q4'), 15999., 843., 1., 2.5,),\n", " ],\n", " columns=['period', 'distance', 'elevation_gain', 'moving_time',\n", " 'elapsed_time'])\n", " # Act\n", " actual = inp.pipe(fillna())\n", " # Assert\n", " assert actual.equals(expected)\n", " \n", " def test_fillna_empty(self):\n", " # Arrange\n", " expected = pd.DataFrame(columns=['period', 'distance', 'elevation_gain',\n", " 'moving_time', 'elapsed_time'])\n", " # Act\n", " actual = expected.pipe(densify())\n", " # Assert\n", " assert actual.equals(expected)\n", "\n", "\n", "RunTest(fillna_test)\n" ], "execution_count": null, "outputs": [] }, { "cell_type": "code", "metadata": { "id": "SY5WggBE1nBO", "colab_type": "code", "cellView": "form", "colab": {} }, "source": [ "#@title aggregate_over_periods\n", "\n", "def aggregate_over_periods(freq):\n", " \"\"\"Aggregates activities over periods at a given frequency.\n", "\n", " column type\n", " ----------------------------------\n", " period pandas.Period\n", " distance int64\n", " moving_time pandas.Timedelta\n", " elapsed_time pandas.Timedelta\n", " elevation_gain int64 \n", " \"\"\"\n", " def by_period(df):\n", " cp = df.set_index('start_date').resample(freq, kind='period').agg(\n", " {\n", " 'distance': np.sum,\n", " 'elevation_gain': np.sum,\n", " 'moving_time': np.sum,\n", " 'elapsed_time': np.sum,\n", " })\n", " cp.index.rename('period', inplace=True)\n", " return cp\n", "\n", " def sort(df):\n", " return df.sort_values('period', ascending=True)\n", "\n", " return lambda df: (df\n", " .pipe(by_period)\n", " .reset_index()\n", " .pipe(densify())\n", " .pipe(fillna())\n", " .pipe(sort)\n", " .reset_index(drop=True))\n", " \n", "\n", "\n", "class aggregate_over_periods_test(unittest.TestCase):\n", "\n", " def test_aggregate_over_periods(self):\n", " # Arrange\n", " inp = pd.DataFrame.from_records(\n", " [\n", " (pd.Timestamp('2019-01-03 13:01:00 UTC'), 24200.0, 232.0, pd.Timedelta('2 hours'), pd.Timedelta('3 hours'),),\n", " (pd.Timestamp('2019-01-04 17:31:00 UTC'), 10000.0, 100.0, pd.Timedelta('1 hours'), pd.Timedelta('2 hours'),),\n", " (pd.Timestamp('2019-02-12 09:22:00 UTC'), 5000.0, 20.0, pd.Timedelta('40 min'), pd.Timedelta('45 min'),),\n", " ],\n", " columns=['start_date', 'distance', 'elevation_gain', 'moving_time',\n", " 'elapsed_time'])\n", " expected = pd.DataFrame.from_records(\n", " [\n", " (pd.Period('2019-01'), 34200.0, 332.0, pd.Timedelta('3 hours'), pd.Timedelta('5 hours'),),\n", " (pd.Period('2019-02'), 5000.0, 20.0, pd.Timedelta('40 min'), pd.Timedelta('45 min'),),\n", " ],\n", " columns=['period', 'distance', 'elevation_gain', 'moving_time',\n", " 'elapsed_time'])\n", " # Act\n", " actual = inp.pipe(aggregate_over_periods('M'))\n", " # Assert\n", " assert actual.equals(expected)\n", " \n", "\n", "RunTest(aggregate_over_periods_test)" ], "execution_count": null, "outputs": [] }, { "cell_type": "markdown", "metadata": { "id": "8QnECJwm--6p", "colab_type": "text" }, "source": [ "## Run" ] }, { "cell_type": "code", "metadata": { "id": "gFPscJzl-74S", "colab_type": "code", "cellView": "form", "colab": {} }, "source": [ "#@title Build activities (tidy data) table\n", "\n", "_activities = _raw_activities.pipe(activities_table())\n", "\n", "_activities.tail()" ], "execution_count": null, "outputs": [] }, { "cell_type": "markdown", "metadata": { "id": "L3sq8pC0jy3F", "colab_type": "text" }, "source": [ "# Report" ] }, { "cell_type": "markdown", "metadata": { "id": "lhHkpcsE_Rwg", "colab_type": "text" }, "source": [ "## Functions" ] }, { "cell_type": "code", "metadata": { "id": "kWsyWLKx_I1P", "colab_type": "code", "cellView": "form", "colab": {} }, "source": [ "#@title plot_totals_over_time\n", "\n", "def plot_totals_over_time(activities,\n", " metric,\n", " time_frame,\n", " granularity):\n", " # Convert the user selected granularity into something that pandas\n", " # understands.\n", " freq = {\n", " 'years': 'Y',\n", " 'quarters': 'Q',\n", " 'months': 'M',\n", " 'weeks': 'W',\n", " 'days': 'D',\n", " }[granularity]\n", "\n", " # Determine the start and the end of the time frame based on the user\n", " # selection.\n", " activity_periods = activities.pipe(aggregate_over_periods(freq=freq))\n", " start = None\n", " end = None\n", " now = pd.Timestamp.now()\n", " if time_frame == \"all data\":\n", " start = activity_periods['period'].min()\n", " end = activity_periods['period'].max()\n", " elif time_frame == \"year-to-date\":\n", " start = pd.Timestamp(year=now.year,\n", " month=1,\n", " day=1)\n", " end = now\n", " elif time_frame == \"month-to-date\":\n", " start = pd.Timestamp(\n", " year=now.year,\n", " month=now.month,\n", " day=1)\n", " end = now\n", " elif time_frame == \"this year\":\n", " start = pd.Timestamp(\n", " year=now.year,\n", " month=1,\n", " day=1)\n", " end = (start\n", " + pd.DateOffset(years=1)\n", " - pd.DateOffset(days=1))\n", " elif time_frame == \"this month\":\n", " start = pd.Timestamp(\n", " year=now.year,\n", " month=now.month,\n", " day=1)\n", " end = (start\n", " + pd.DateOffset(months=1)\n", " - pd.DateOffset(days=1))\n", " elif time_frame == \"last 30 days\":\n", " start = pd.Timestamp.now() - pd.Timedelta('30 days')\n", " end = pd.Timestamp.now()\n", " elif time_frame == \"last 365 days\":\n", " start = pd.Timestamp.now() - pd.Timedelta('365 days')\n", " end = pd.Timestamp.now()\n", " elif time_frame == \"last 730 days\":\n", " start = pd.Timestamp.now() - pd.Timedelta('730 days')\n", " end = pd.Timestamp.now()\n", " elif time_frame == \"last 1095 days\":\n", " start = pd.Timestamp.now() - pd.Timedelta('1095 days')\n", " end = pd.Timestamp.now()\n", " else:\n", " assert False, \"not implemented\"\n", " assert start\n", " assert end\n", "\n", " ax = (activity_periods\n", " .pipe(align_to_period(\n", " pd.period_range(\n", " start=start,\n", " end=end,\n", " freq=freq)))\n", " .set_index('period')[metric]\n", " .plot(kind='bar'))\n", " \n", " ax.set_title({\n", " 'distance': 'distance',\n", " 'elevation_gain': 'elevation gain',\n", " 'elapsed_time': 'elapsed time',\n", " 'moving_time': 'moving time',\n", " }[metric])\n", " ax.set_ylabel({\n", " 'distance': 'kilometers',\n", " 'elevation_gain': 'meters',\n", " 'elapsed_time': 'hours',\n", " 'moving_time': 'hours',\n", " }[metric])\n", " ax.set_xlabel('')\n", "\n" ], "execution_count": null, "outputs": [] }, { "cell_type": "markdown", "metadata": { "id": "qTTy-RZtVcEk", "colab_type": "text" }, "source": [ "## Totals over time\n", "\n", "This report allows you to plot the totals for a particular **metric** over a particular **time frame** at a specified **granularity**." ] }, { "cell_type": "code", "metadata": { "colab_type": "code", "cellView": "form", "id": "FJyljzy-LPxY", "colab": {} }, "source": [ "METRIC = \"elevation_gain\" #@param [\"distance\", \"elevation_gain\", \"moving_time\", \"elapsed_time\"]\n", "TIME_FRAME = \"last 730 days\" #@param [\"all data\", \"year-to-date\", \"month-to-date\", \"this year\", \"this quarter\", \"this month\", \"last 30 days\", \"last 365 days\", \"last 730 days\", \"last 1095 days\"]\n", "GRANULARITY = \"quarters\" #@param [\"years\", \"quarters\", \"months\", \"weeks\", \"days\"]\n", "WIDTH_SCALING_FACTOR = 1 #@param {type: \"number\"}\n", "\n", "_width = 700.0 / (936.0 / 13.0)\n", "plt.figure(figsize=(WIDTH_SCALING_FACTOR * _width, 3));\n", "plot_totals_over_time(_activities,\n", " metric=METRIC,\n", " time_frame=TIME_FRAME,\n", " granularity=GRANULARITY);\n", "plt.show();\n" ], "execution_count": null, "outputs": [] } ] }