{ "cells": [ { "cell_type": "markdown", "metadata": {}, "source": [ "# Temporal Resampling and gapfilling" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "> The package `otbApplication` is not installed in SISE environment. You must activate SISE-3.7 environnment by launching the following command in Anaconda Prompt : `conda activate sise-3.7`" ] }, { "cell_type": "code", "execution_count": 16, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "All libraries successfully imported!\n" ] } ], "source": [ "import glob, os, subprocess\n", "import pandas as pd\n", "import otbApplication as otb\n", "from pathlib import Path\n", "import datetime\n", "import rasterio\n", "import numpy as np\n", "\n", "print('All libraries successfully imported!')" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "**Set directory**" ] }, { "cell_type": "code", "execution_count": 17, "metadata": {}, "outputs": [], "source": [ "computer_path = '/export/miro/ndeffense/LBRAT2104/'\n", "grp_nb = '99'\n", "\n", "# Directory for all work files\n", "work_path = f'{computer_path}STUDENTS/GROUP_{grp_nb}/TP/'\n", "\n", "clipped_path = f'{work_path}2_L2A_CLIPPED/'\n", "gapfilled_path = f'{work_path}4_L2A_GAPFILLED/'\n", "\n", "Path(gapfilled_path).mkdir(parents=True, exist_ok=True)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "**Parameters**" ] }, { "cell_type": "code", "execution_count": 18, "metadata": {}, "outputs": [], "source": [ "band_list = ['B02','B03']\n", "\n", "format = '%Y%m%d' # format : YYYYMMDD\n", "begin_date = 7\n", "\n", "# Distance between interpolated dates\n", "frequency = '1m'\n", "\n", "# Interpolation type\n", "interp_type = 'linear' # 'spline'\n", "\n", "flag_otbAPI = False\n" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Prepare data\n", "### Get dataframe with all inputs" ] }, { "cell_type": "code", "execution_count": 19, "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
datedate_strSCLB02B03
02020-01-1620200116/export/miro/ndeffense/LBRAT2104/STUDENTS/GROU.../export/miro/ndeffense/LBRAT2104/STUDENTS/GROU.../export/miro/ndeffense/LBRAT2104/STUDENTS/GROU...
12020-02-1220200212/export/miro/ndeffense/LBRAT2104/STUDENTS/GROU.../export/miro/ndeffense/LBRAT2104/STUDENTS/GROU.../export/miro/ndeffense/LBRAT2104/STUDENTS/GROU...
22020-03-1620200316/export/miro/ndeffense/LBRAT2104/STUDENTS/GROU.../export/miro/ndeffense/LBRAT2104/STUDENTS/GROU.../export/miro/ndeffense/LBRAT2104/STUDENTS/GROU...
32020-04-1720200417/export/miro/ndeffense/LBRAT2104/STUDENTS/GROU.../export/miro/ndeffense/LBRAT2104/STUDENTS/GROU.../export/miro/ndeffense/LBRAT2104/STUDENTS/GROU...
42020-05-2020200520/export/miro/ndeffense/LBRAT2104/STUDENTS/GROU.../export/miro/ndeffense/LBRAT2104/STUDENTS/GROU.../export/miro/ndeffense/LBRAT2104/STUDENTS/GROU...
52020-06-2120200621/export/miro/ndeffense/LBRAT2104/STUDENTS/GROU.../export/miro/ndeffense/LBRAT2104/STUDENTS/GROU.../export/miro/ndeffense/LBRAT2104/STUDENTS/GROU...
62020-07-1920200719/export/miro/ndeffense/LBRAT2104/STUDENTS/GROU.../export/miro/ndeffense/LBRAT2104/STUDENTS/GROU.../export/miro/ndeffense/LBRAT2104/STUDENTS/GROU...
72020-08-1320200813/export/miro/ndeffense/LBRAT2104/STUDENTS/GROU.../export/miro/ndeffense/LBRAT2104/STUDENTS/GROU.../export/miro/ndeffense/LBRAT2104/STUDENTS/GROU...
82020-09-1420200914/export/miro/ndeffense/LBRAT2104/STUDENTS/GROU.../export/miro/ndeffense/LBRAT2104/STUDENTS/GROU.../export/miro/ndeffense/LBRAT2104/STUDENTS/GROU...
92020-10-1920201019/export/miro/ndeffense/LBRAT2104/STUDENTS/GROU.../export/miro/ndeffense/LBRAT2104/STUDENTS/GROU.../export/miro/ndeffense/LBRAT2104/STUDENTS/GROU...
102020-11-1820201118/export/miro/ndeffense/LBRAT2104/STUDENTS/GROU.../export/miro/ndeffense/LBRAT2104/STUDENTS/GROU.../export/miro/ndeffense/LBRAT2104/STUDENTS/GROU...
112020-12-1820201218/export/miro/ndeffense/LBRAT2104/STUDENTS/GROU.../export/miro/ndeffense/LBRAT2104/STUDENTS/GROU.../export/miro/ndeffense/LBRAT2104/STUDENTS/GROU...
\n", "
" ], "text/plain": [ " date date_str SCL \\\n", "0 2020-01-16 20200116 /export/miro/ndeffense/LBRAT2104/STUDENTS/GROU... \n", "1 2020-02-12 20200212 /export/miro/ndeffense/LBRAT2104/STUDENTS/GROU... \n", "2 2020-03-16 20200316 /export/miro/ndeffense/LBRAT2104/STUDENTS/GROU... \n", "3 2020-04-17 20200417 /export/miro/ndeffense/LBRAT2104/STUDENTS/GROU... \n", "4 2020-05-20 20200520 /export/miro/ndeffense/LBRAT2104/STUDENTS/GROU... \n", "5 2020-06-21 20200621 /export/miro/ndeffense/LBRAT2104/STUDENTS/GROU... \n", "6 2020-07-19 20200719 /export/miro/ndeffense/LBRAT2104/STUDENTS/GROU... \n", "7 2020-08-13 20200813 /export/miro/ndeffense/LBRAT2104/STUDENTS/GROU... \n", "8 2020-09-14 20200914 /export/miro/ndeffense/LBRAT2104/STUDENTS/GROU... \n", "9 2020-10-19 20201019 /export/miro/ndeffense/LBRAT2104/STUDENTS/GROU... \n", "10 2020-11-18 20201118 /export/miro/ndeffense/LBRAT2104/STUDENTS/GROU... \n", "11 2020-12-18 20201218 /export/miro/ndeffense/LBRAT2104/STUDENTS/GROU... \n", "\n", " B02 \\\n", "0 /export/miro/ndeffense/LBRAT2104/STUDENTS/GROU... \n", "1 /export/miro/ndeffense/LBRAT2104/STUDENTS/GROU... \n", "2 /export/miro/ndeffense/LBRAT2104/STUDENTS/GROU... \n", "3 /export/miro/ndeffense/LBRAT2104/STUDENTS/GROU... \n", "4 /export/miro/ndeffense/LBRAT2104/STUDENTS/GROU... \n", "5 /export/miro/ndeffense/LBRAT2104/STUDENTS/GROU... \n", "6 /export/miro/ndeffense/LBRAT2104/STUDENTS/GROU... \n", "7 /export/miro/ndeffense/LBRAT2104/STUDENTS/GROU... \n", "8 /export/miro/ndeffense/LBRAT2104/STUDENTS/GROU... \n", "9 /export/miro/ndeffense/LBRAT2104/STUDENTS/GROU... \n", "10 /export/miro/ndeffense/LBRAT2104/STUDENTS/GROU... \n", "11 /export/miro/ndeffense/LBRAT2104/STUDENTS/GROU... \n", "\n", " B03 \n", "0 /export/miro/ndeffense/LBRAT2104/STUDENTS/GROU... \n", "1 /export/miro/ndeffense/LBRAT2104/STUDENTS/GROU... \n", "2 /export/miro/ndeffense/LBRAT2104/STUDENTS/GROU... \n", "3 /export/miro/ndeffense/LBRAT2104/STUDENTS/GROU... \n", "4 /export/miro/ndeffense/LBRAT2104/STUDENTS/GROU... \n", "5 /export/miro/ndeffense/LBRAT2104/STUDENTS/GROU... \n", "6 /export/miro/ndeffense/LBRAT2104/STUDENTS/GROU... \n", "7 /export/miro/ndeffense/LBRAT2104/STUDENTS/GROU... \n", "8 /export/miro/ndeffense/LBRAT2104/STUDENTS/GROU... \n", "9 /export/miro/ndeffense/LBRAT2104/STUDENTS/GROU... \n", "10 /export/miro/ndeffense/LBRAT2104/STUDENTS/GROU... \n", "11 /export/miro/ndeffense/LBRAT2104/STUDENTS/GROU... " ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "dict_list = []\n", "\n", "for im in glob.glob(f'{clipped_path}*SCL*10m*.tif'):\n", " \n", " date_str = os.path.basename(im)[begin_date:begin_date+8]\n", "\n", " date_im = datetime.datetime.strptime(date_str, format).date()\n", " dict_list.append({'date': date_im,\n", " 'date_str': date_str,\n", " 'SCL': im})\n", "\n", "df = pd.DataFrame.from_dict(dict_list).sort_values('date')\n", "\n", "\n", "for band in band_list:\n", "\n", " df[band] = \"\" # Create empty column in dataframe\n", "\n", " for im in glob.glob(f'{clipped_path}*{band}*.tif'):\n", " \n", " date_str = os.path.basename(im)[begin_date:begin_date+8]\n", "\n", " date_im = datetime.datetime.strptime(date_str, format).date()\n", "\n", " df.loc[df.date == date_im, band] = im\n", "\n", "display(df)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Modify and concatenate masks " ] }, { "cell_type": "code", "execution_count": 20, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "(570, 986, 12)\n", "There are 12 masks\n" ] } ], "source": [ "mask_concat_file = f'{gapfilled_path}SCL_timeserie.tif'\n", " \n", "list_src_arr = []\n", "\n", "for i, row in df.iterrows():\n", "\n", " im_file = row['SCL']\n", "\n", " src = rasterio.open(im_file, \"r\")\n", "\n", " out_meta = src.meta\n", " out_meta.update(count=len(df))\n", "\n", " # Read file as numpy array\n", " SCL = src.read(1)\n", " src.close()\n", "\n", " SCL[SCL == 0] = 1 # No data\n", " SCL[SCL == 1] = 1 # Saturated or defective\n", " SCL[SCL == 2] = 1 # Dark area pixels\n", " SCL[SCL == 3] = 1 # Cloud shadows\n", " SCL[SCL == 4] = 0 # Vegetation\n", " SCL[SCL == 5] = 0 # Not vegetated\n", " SCL[SCL == 6] = 0 # Water\n", " SCL[SCL == 7] = 0 # Unclassified\n", " SCL[SCL == 8] = 1 # Cloud medium probability\n", " SCL[SCL == 9] = 1 # Cloud high probability\n", " SCL[SCL == 10] = 1 # Thin cirrus\n", " SCL[SCL == 11] = 1 # Snow\n", "\n", " list_src_arr.append(SCL)\n", "\n", "mask_arr_stack = np.dstack(list_src_arr)#.astype(np.float32)\n", "\n", "print(mask_arr_stack.shape)\n", "print(f'There are {mask_arr_stack.shape[2]} masks')\n", "\n", "with rasterio.open(mask_concat_file, \"w\", **out_meta) as dest:\n", " for band_nr, src in enumerate(list_src_arr, start=1):\n", " dest.write(src, band_nr)\n" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Get input dates" ] }, { "cell_type": "code", "execution_count": 21, "metadata": {}, "outputs": [], "source": [ "input_dates_txt = f'{gapfilled_path}input_dates.txt'\n", "\n", "input_dates_list = df['date_str'].to_list()\n", "\n", "with open(input_dates_txt, 'w') as f:\n", " for i in range(0,len(input_dates_list)):\n", " f.write(input_dates_list[i] + '\\n')" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Get output dates" ] }, { "cell_type": "code", "execution_count": 22, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "20200131\n", "20200229\n", "20200331\n", "20200430\n", "20200531\n", "20200630\n", "20200731\n", "20200831\n", "20200930\n", "20201031\n", "20201130\n" ] } ], "source": [ "output_dates_txt = f'{gapfilled_path}output_dates.txt'\n", "\n", "start_date = df['date'].iloc[0]\n", "end_date = df['date'].iloc[-1]\n", "\n", "output_dates_list = pd.date_range(start_date,end_date,freq=frequency).strftime('%Y%m%d').to_list()\n", "\n", "with open(output_dates_txt, 'w') as f:\n", " for i in range(0,len(output_dates_list)):\n", " print(output_dates_list[i])\n", " f.write(output_dates_list[i] + '\\n')" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Concatenate reflectance timeseries" ] }, { "cell_type": "code", "execution_count": 27, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "otbcli_ConcatenateImages -il /export/miro/ndeffense/LBRAT2104/STUDENTS/GROUP_99/TP/2_L2A_CLIPPED/T31UFS_20200116T105309_B02_10m_ROI.tif /export/miro/ndeffense/LBRAT2104/STUDENTS/GROUP_99/TP/2_L2A_CLIPPED/T31UFS_20200212T104049_B02_10m_ROI.tif /export/miro/ndeffense/LBRAT2104/STUDENTS/GROUP_99/TP/2_L2A_CLIPPED/T31UFS_20200316T104709_B02_10m_ROI.tif /export/miro/ndeffense/LBRAT2104/STUDENTS/GROUP_99/TP/2_L2A_CLIPPED/T31UFS_20200417T104021_B02_10m_ROI.tif /export/miro/ndeffense/LBRAT2104/STUDENTS/GROUP_99/TP/2_L2A_CLIPPED/T31UFS_20200520T105031_B02_10m_ROI.tif /export/miro/ndeffense/LBRAT2104/STUDENTS/GROUP_99/TP/2_L2A_CLIPPED/T31UFS_20200621T103629_B02_10m_ROI.tif /export/miro/ndeffense/LBRAT2104/STUDENTS/GROUP_99/TP/2_L2A_CLIPPED/T31UFS_20200719T105031_B02_10m_ROI.tif /export/miro/ndeffense/LBRAT2104/STUDENTS/GROUP_99/TP/2_L2A_CLIPPED/T31UFS_20200813T104629_B02_10m_ROI.tif /export/miro/ndeffense/LBRAT2104/STUDENTS/GROUP_99/TP/2_L2A_CLIPPED/T31UFS_20200914T104031_B02_10m_ROI.tif /export/miro/ndeffense/LBRAT2104/STUDENTS/GROUP_99/TP/2_L2A_CLIPPED/T31UFS_20201019T103959_B02_10m_ROI.tif /export/miro/ndeffense/LBRAT2104/STUDENTS/GROUP_99/TP/2_L2A_CLIPPED/T31UFS_20201118T104329_B02_10m_ROI.tif /export/miro/ndeffense/LBRAT2104/STUDENTS/GROUP_99/TP/2_L2A_CLIPPED/T31UFS_20201218T104349_B02_10m_ROI.tif -out /export/miro/ndeffense/LBRAT2104/STUDENTS/GROUP_99/TP/4_L2A_GAPFILLED/B02_timeserie.tif\n", "-------------------------------------------\n", "otbcli_ConcatenateImages -il /export/miro/ndeffense/LBRAT2104/STUDENTS/GROUP_99/TP/2_L2A_CLIPPED/T31UFS_20200116T105309_B03_10m_ROI.tif /export/miro/ndeffense/LBRAT2104/STUDENTS/GROUP_99/TP/2_L2A_CLIPPED/T31UFS_20200212T104049_B03_10m_ROI.tif /export/miro/ndeffense/LBRAT2104/STUDENTS/GROUP_99/TP/2_L2A_CLIPPED/T31UFS_20200316T104709_B03_10m_ROI.tif /export/miro/ndeffense/LBRAT2104/STUDENTS/GROUP_99/TP/2_L2A_CLIPPED/T31UFS_20200417T104021_B03_10m_ROI.tif /export/miro/ndeffense/LBRAT2104/STUDENTS/GROUP_99/TP/2_L2A_CLIPPED/T31UFS_20200520T105031_B03_10m_ROI.tif /export/miro/ndeffense/LBRAT2104/STUDENTS/GROUP_99/TP/2_L2A_CLIPPED/T31UFS_20200621T103629_B03_10m_ROI.tif /export/miro/ndeffense/LBRAT2104/STUDENTS/GROUP_99/TP/2_L2A_CLIPPED/T31UFS_20200719T105031_B03_10m_ROI.tif /export/miro/ndeffense/LBRAT2104/STUDENTS/GROUP_99/TP/2_L2A_CLIPPED/T31UFS_20200813T104629_B03_10m_ROI.tif /export/miro/ndeffense/LBRAT2104/STUDENTS/GROUP_99/TP/2_L2A_CLIPPED/T31UFS_20200914T104031_B03_10m_ROI.tif /export/miro/ndeffense/LBRAT2104/STUDENTS/GROUP_99/TP/2_L2A_CLIPPED/T31UFS_20201019T103959_B03_10m_ROI.tif /export/miro/ndeffense/LBRAT2104/STUDENTS/GROUP_99/TP/2_L2A_CLIPPED/T31UFS_20201118T104329_B03_10m_ROI.tif /export/miro/ndeffense/LBRAT2104/STUDENTS/GROUP_99/TP/2_L2A_CLIPPED/T31UFS_20201218T104349_B03_10m_ROI.tif -out /export/miro/ndeffense/LBRAT2104/STUDENTS/GROUP_99/TP/4_L2A_GAPFILLED/B03_timeserie.tif\n", "-------------------------------------------\n" ] } ], "source": [ "for band in band_list:\n", "\n", " im_concat_file = f'{gapfilled_path}{band}_timeserie.tif'\n", " \n", " im_path_list = df[band].tolist()\n", " \n", " if flag_otbAPI:\n", " app = otb.Registry.CreateApplication(\"ConcatenateImages\")\n", "\n", " app.SetParameterStringList(\"il\", im_path_list)\n", " app.SetParameterString(\"out\", im_concat_file)\n", "\n", " app.ExecuteAndWriteOutput()\n", "\n", " else:\n", " cmd = 'otbcli_ConcatenateImages'\n", " cmd += f' -il {\" \".join(im_path_list)}'\n", " cmd += f' -out {im_concat_file}'\n", "\n", " print(cmd)\n", " subprocess.call(cmd, shell=True)\n", " print('-------------------------------------------')" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Temporal Resampling and Gapfilling" ] }, { "cell_type": "code", "execution_count": 26, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "otbcli_ImageTimeSeriesGapFilling -in /export/miro/ndeffense/LBRAT2104/STUDENTS/GROUP_99/TP/4_L2A_GAPFILLED/B02_timeserie.tif -mask /export/miro/ndeffense/LBRAT2104/STUDENTS/GROUP_99/TP/4_L2A_GAPFILLED/SCL_timeserie.tif -out /export/miro/ndeffense/LBRAT2104/STUDENTS/GROUP_99/TP/4_L2A_GAPFILLED/B02_timeserie_gapfilled_1m.tif -comp 1 -it linear -id /export/miro/ndeffense/LBRAT2104/STUDENTS/GROUP_99/TP/4_L2A_GAPFILLED/input_dates.txt -od /export/miro/ndeffense/LBRAT2104/STUDENTS/GROUP_99/TP/4_L2A_GAPFILLED/output_dates.txt\n", "-------------------------------------------\n", "otbcli_ImageTimeSeriesGapFilling -in /export/miro/ndeffense/LBRAT2104/STUDENTS/GROUP_99/TP/4_L2A_GAPFILLED/B03_timeserie.tif -mask /export/miro/ndeffense/LBRAT2104/STUDENTS/GROUP_99/TP/4_L2A_GAPFILLED/SCL_timeserie.tif -out /export/miro/ndeffense/LBRAT2104/STUDENTS/GROUP_99/TP/4_L2A_GAPFILLED/B03_timeserie_gapfilled_1m.tif -comp 1 -it linear -id /export/miro/ndeffense/LBRAT2104/STUDENTS/GROUP_99/TP/4_L2A_GAPFILLED/input_dates.txt -od /export/miro/ndeffense/LBRAT2104/STUDENTS/GROUP_99/TP/4_L2A_GAPFILLED/output_dates.txt\n", "-------------------------------------------\n" ] } ], "source": [ "for band in band_list:\n", "\n", " im_concat_file = f'{gapfilled_path}{band}_timeserie.tif'\n", "\n", " gapfilled_file = f'{gapfilled_path}{band}_timeserie_gapfilled_{frequency}.tif'\n", "\n", " if flag_otbAPI:\n", " app = otb.Registry.CreateApplication(\"ImageTimeSeriesGapFilling\")\n", "\n", " app.SetParameterString(\"in\", im_concat_file)\n", " app.SetParameterString(\"mask\", mask_concat_file)\n", " app.SetParameterString(\"out\", gapfilled_file)\n", " app.SetParameterInt(\"comp\", 1)\n", " app.SetParameterString(\"it\", interp_type)\n", " app.SetParameterString(\"id\", input_dates_txt)\n", " app.SetParameterString(\"od\", output_dates_txt)\n", "\n", " app.ExecuteAndWriteOutput()\n", " \n", " else:\n", " cmd = 'otbcli_ImageTimeSeriesGapFilling'\n", " cmd += f' -in {im_concat_file}'\n", " cmd += f' -mask {mask_concat_file}'\n", " cmd += f' -out {gapfilled_file}'\n", " cmd += f' -comp 1'\n", " cmd += f' -it {interp_type}'\n", " cmd += f' -id {input_dates_txt}'\n", " cmd += f' -od {output_dates_txt}'\n", "\n", " print(cmd)\n", " subprocess.call(cmd, shell=True)\n", " print('-------------------------------------------')" ] }, { "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ "## Display gapfilled timeserie\n", "\n", "**TO DO**" ] } ], "metadata": { "kernelspec": { "display_name": "Python 3.6.12 64-bit", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.10.8" }, "orig_nbformat": 4, "vscode": { "interpreter": { "hash": "31f2aee4e71d21fbe5cf8b01ff0e069b9275f58929596ceb00d14d90e3e16cd6" } } }, "nbformat": 4, "nbformat_minor": 2 }