diff --git a/.devcontainer.json b/.devcontainer.json new file mode 100644 index 00000000..9c87d96b --- /dev/null +++ b/.devcontainer.json @@ -0,0 +1,29 @@ +{ + "containerEnv": { + "DISPLAY": "${env:DISPLAY}", + "GLOG_logtostderr": "1", + "PYTHONPATH": "/src" + }, + "dockerFile": "./Dockerfile", + "extensions": [ + "ms-vscode.cmake-tools", + "twxs.cmake", + "ms-python.python", + "ms-vscode.cpptools", + "ms-azuretools.vscode-docker" + ], + "mounts": ["source=/,target=/host,type=bind"], + "runArgs": [ + "--init", + "--rm", + "-ti", + "--gpus=all", + "--cap-add=SYS_PTRACE", + "--ipc=host", + "--net=host", + "--security-opt", + "apparmor:unconfined" + ], + "workspaceFolder": "/src", + "workspaceMount": "source=${localWorkspaceFolder},target=/src,type=bind,consistency=cached" +} diff --git a/Dockerfile b/Dockerfile index 7c365cc1..55b8df55 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,98 +1,73 @@ -# From https://github.com/ufoym/deepo/blob/master/docker/Dockerfile.pytorch-py36-cu90 +# syntax = docker/dockerfile:1.0-experimental -# ================================================================== -# module list -# ------------------------------------------------------------------ -# python 3.6 (apt) -# pytorch latest (pip) -# ================================================================== +FROM nvidia/cudagl:10.2-devel-ubuntu18.04 -FROM nvidia/cuda:9.0-cudnn7-devel-ubuntu16.04 -RUN APT_INSTALL="apt-get install -y --no-install-recommends" && \ - PIP_INSTALL="python -m pip --no-cache-dir install --upgrade" && \ - GIT_CLONE="git clone --depth 10" && \ - rm -rf /var/lib/apt/lists/* \ - /etc/apt/sources.list.d/cuda.list \ - /etc/apt/sources.list.d/nvidia-ml.list && \ - apt-get update && \ -# ================================================================== -# tools -# ------------------------------------------------------------------ - DEBIAN_FRONTEND=noninteractive $APT_INSTALL \ - build-essential \ - ca-certificates \ - cmake \ - wget \ - git \ - vim \ - fish \ - libsparsehash-dev \ - && \ -# ================================================================== -# python -# ------------------------------------------------------------------ - DEBIAN_FRONTEND=noninteractive $APT_INSTALL \ - software-properties-common \ - && \ - add-apt-repository ppa:deadsnakes/ppa && \ - apt-get update && \ - DEBIAN_FRONTEND=noninteractive $APT_INSTALL \ - python3.6 \ - python3.6-dev \ - && \ - wget -O ~/get-pip.py \ - https://bootstrap.pypa.io/get-pip.py && \ - python3.6 ~/get-pip.py && \ - ln -s /usr/bin/python3.6 /usr/local/bin/python3 && \ - ln -s /usr/bin/python3.6 /usr/local/bin/python && \ - $PIP_INSTALL \ - setuptools \ - && \ - $PIP_INSTALL \ - numpy \ - scipy \ - matplotlib \ - Cython \ - && \ -# ================================================================== -# pytorch -# ------------------------------------------------------------------ - $PIP_INSTALL \ - torch_nightly -f \ - https://download.pytorch.org/whl/nightly/cu90/torch_nightly.html \ - && \ - $PIP_INSTALL \ - torchvision_nightly \ - && \ -# ================================================================== -# config & cleanup -# ------------------------------------------------------------------ - ldconfig && \ - apt-get clean && \ - apt-get autoremove && \ - rm -rf /var/lib/apt/lists/* /tmp/* ~/* +ARG UBUNTU_MAJOR_MINOR_VERSION="18.04" +ARG CUDA_MAJOR_MINOR_VERSION="10.2" +ARG CUDNN_MAJOR_VERSION="8" +ARG CUDNN_MAJOR_MINOR_VERSION="${CUDNN_MAJOR_VERSION}.0" +ARG CUDNN_VERSION="${CUDNN_MAJOR_MINOR_VERSION}.5.39" +ARG CUDNN_FULL_VERSION="${CUDNN_VERSION}-1+cuda${CUDA_MAJOR_MINOR_VERSION}" -RUN PIP_INSTALL="python -m pip --no-cache-dir install --upgrade" && \ - $PIP_INSTALL \ - shapely fire pybind11 tensorboardX protobuf \ - scikit-image numba pillow +RUN \ + # update packages + apt-get update && apt-get upgrade -y \ + # install additional packages + && DEBIAN_FRONTEND=noninteractive apt-get install --fix-missing --no-install-recommends -y \ + git \ + libboost-all-dev \ + libcudnn${CUDNN_MAJOR_VERSION}=${CUDNN_FULL_VERSION} \ + libcudnn${CUDNN_MAJOR_VERSION}-dev=${CUDNN_FULL_VERSION} \ + git \ + wget \ + # make sure these packages don't get upgraded + && apt-mark hold \ + libcudnn${CUDNN_MAJOR_VERSION} \ + libcudnn${CUDNN_MAJOR_VERSION}-dev \ + # clean up + && rm -rf /var/lib/apt/lists/* -WORKDIR /root -RUN wget https://dl.bintray.com/boostorg/release/1.68.0/source/boost_1_68_0.tar.gz -RUN tar xzvf boost_1_68_0.tar.gz -RUN cp -r ./boost_1_68_0/boost /usr/include -RUN rm -rf ./boost_1_68_0 -RUN rm -rf ./boost_1_68_0.tar.gz -RUN git clone https://github.com/traveller59/second.pytorch.git --depth 10 -RUN git clone https://github.com/traveller59/SparseConvNet.git --depth 10 -RUN cd ./SparseConvNet && python setup.py install && cd .. && rm -rf SparseConvNet -ENV NUMBAPRO_CUDA_DRIVER=/usr/lib/x86_64-linux-gnu/libcuda.so -ENV NUMBAPRO_NVVM=/usr/local/cuda/nvvm/lib64/libnvvm.so -ENV NUMBAPRO_LIBDEVICE=/usr/local/cuda/nvvm/libdevice -ENV PYTHONPATH=/root/second.pytorch +# install miniconda, adding packages to the base environment +ARG MINICONDA_VERSION="py38_4.10.3" +ARG MINICONDA_PACKAGE="Miniconda3-${MINICONDA_VERSION}-Linux-x86_64.sh" +ARG MINICONDA_INSTALL_DIR="/root/miniconda3" +ENV PATH="${MINICONDA_INSTALL_DIR}/bin:${MINICONDA_INSTALL_DIR}/condabin:${PATH}" +WORKDIR /home/docker-user +RUN wget https://repo.anaconda.com/miniconda/${MINICONDA_PACKAGE} \ + && bash ${MINICONDA_PACKAGE} -b -p ${MINICONDA_INSTALL_DIR} \ + && conda update -y -n base -c defaults conda \ + && conda install -c pytorch -c nvidia -c defaults -c conda-forge \ + # packages to be in the base environment + black \ + cmake \ + cudatoolkit=10.2.* \ + numba \ + numpy \ + opencv \ + openexr-python \ + pip=20.3.* \ + protobuf \ + psutil \ + python=3.8.* \ + pytorch=1.9.* \ + scikit-image \ + seaborn \ + tensorboardx \ + torchvision \ + wheel \ + # configure and clean up + && conda init \ + && conda config --set report_errors false \ + && conda clean -tipsy \ + && rm ${MINICONDA_PACKAGE} -VOLUME ["/root/data"] -VOLUME ["/root/model"] -WORKDIR /root/second.pytorch/second +# install some extra pip packages +RUN pip install \ + fire \ + lupa -ENTRYPOINT ["fish"] +# build and install the spconv package +RUN git clone --depth 1 --recursive https://github.com/traveller59/spconv.git \ + && cd spconv \ + && SPCONV_FORCE_BUILD_CUDA=1 python setup.py install \ + && rm -rf spconv diff --git a/README.md b/README.md index 0a5b204a..09056f52 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,21 @@ +# To Run it on linux desktops + +The docker image is saved at `/mnt/nas/quan/docker_images/ce_second_env.tar`. + +The image is also available in the registry. Get it by running: + +```bash +> docker pull nas1.rwc.compoundeye.com:6001/ce_second_env +``` + +To launch the container: + +``` bash +> docker run -it --rm --ipc=host --net=host --gpus all -v /host:/host nas1.rwc.compoundeye.com:6001/ce_second_env:latest bash +> conda activate second +> export PYTHONPATH=${PYTHONPATH}:/host/home/quan/Documents/code/second.pytorch +``` + # SECOND for KITTI/NuScenes object detection (1.6.0 Alpha) SECOND detector. @@ -11,7 +29,7 @@ If you want to train nuscenes dataset, see [this](NUSCENES-GUIDE.md). 2019-4-1: SECOND V1.6.0alpha released: New Data API, [NuScenes](https://www.nuscenes.org) support, [PointPillars](https://github.com/nutonomy/second.pytorch) support, fp16 and multi-gpu support. -2019-3-21: SECOND V1.5.1 (minor improvement and bug fix) released! +2019-3-21: SECOND V1.5.1 (minor improvement and bug fix) released! 2019-1-20: SECOND V1.5 released! Sparse convolution-based network. @@ -98,7 +116,7 @@ If you don't have Anaconda: pip install numba scikit-image scipy pillow ``` -Follow instructions in [spconv](https://github.com/traveller59/spconv) to install spconv. +Follow instructions in [spconv](https://github.com/traveller59/spconv) to install spconv. If you want to train with fp16 mixed precision (train faster in RTX series, Titan V/RTX and Tesla V100, but I only have 1080Ti), you need to install [apex](https://github.com/NVIDIA/apex). @@ -240,13 +258,13 @@ python ./pytorch/train.py evaluate --config_path=./configs/car.fhd.config --mode You can download pretrained models in [google drive](https://drive.google.com/open?id=1YOpgRkBgmSAJwMknoXmitEArNitZz63C). The ```car_fhd``` model is corresponding to car.fhd.config. -Note that this pretrained model is trained before a bug of sparse convolution fixed, so the eval result may slightly worse. +Note that this pretrained model is trained before a bug of sparse convolution fixed, so the eval result may slightly worse. ## Docker (Deprecated. I can't push docker due to network problem.) You can use a prebuilt docker for testing: ``` -docker pull scrin/second-pytorch +docker pull scrin/second-pytorch ``` Then run: ``` diff --git a/dump_csi.sh b/dump_csi.sh new file mode 100644 index 00000000..f4274328 --- /dev/null +++ b/dump_csi.sh @@ -0,0 +1,17 @@ +#! /bin/bash +# +# run reconstruction batch on a CSI dataset to produce output for tracking testing +# +# Run this script in an appropriate docker container. +# +# > dump_csi.sh /path/to/csi/dataset /path/to/put/dump +# +src_dir=$1 +tgt_dir=$2 +/build/reconstruction/reconstruction_batch \ + --settings-file /src/config/demo_app.info \ + --csi-dir ${src_dir} \ + -Svolumetric_settings.block_store_path=${tgt_dir} \ + -Straining_img_blocking=true \ + -Straining_min_interval=1 \ + -Straining_max_interval=1 diff --git a/second/configs/car.fhd.config b/second/configs/car.fhd.config index 08fe206b..1b854158 100644 --- a/second/configs/car.fhd.config +++ b/second/configs/car.fhd.config @@ -118,11 +118,11 @@ model: { train_input_reader: { dataset: { dataset_class_name: "KittiDataset" - kitti_info_path: "/media/yy/960evo/datasets/kitti/kitti_infos_train.pkl" - kitti_root_path: "/media/yy/960evo/datasets/kitti" + kitti_info_path: "/host/ssd/quan/data/tmp/kitti3d_second/kitti_infos_train.pkl" + kitti_root_path: "/host/ssd/quan/data/tmp/kitti3d_second" } - batch_size: 8 + batch_size: 4 preprocess: { max_number_of_voxels: 17000 shuffle_points: true @@ -145,7 +145,7 @@ train_input_reader: { random_flip_y: true remove_environment: false database_sampler { - database_info_path: "/media/yy/960evo/datasets/kitti/kitti_dbinfos_train.pkl" + database_info_path: "/host/ssd/quan/data/tmp/kitti3d_second/kitti_dbinfos_train.pkl" sample_groups { name_to_max_num { key: "Car" @@ -196,7 +196,7 @@ train_config: { save_checkpoints_secs : 1800 # half hour save_summary_steps : 10 - enable_mixed_precision: false + enable_mixed_precision: false loss_scale_factor: -1 clear_metrics_every_epoch: true } @@ -204,9 +204,9 @@ train_config: { eval_input_reader: { dataset: { dataset_class_name: "KittiDataset" - kitti_info_path: "/media/yy/960evo/datasets/kitti/kitti_infos_val.pkl" - # kitti_info_path: "/media/yy/960evo/datasets/kitti/kitti_infos_test.pkl" - kitti_root_path: "/media/yy/960evo/datasets/kitti" + kitti_info_path: "/host/ssd/quan/data/tmp/kitti3d_second/kitti_infos_val.pkl" + # kitti_info_path: "/host/ssd/quan/data/tmp/kitti3d_second/kitti_infos_test.pkl" + kitti_root_path: "/host/ssd/quan/data/tmp/kitti3d_second" } batch_size: 8 preprocess: { @@ -216,4 +216,4 @@ eval_input_reader: { anchor_area_threshold: -1 remove_environment: false } -} \ No newline at end of file +} diff --git a/second/configs/pointpillars/car/xyres_16.config b/second/configs/pointpillars/car/xyres_16.config index ab040981..f2d8cdaf 100644 --- a/second/configs/pointpillars/car/xyres_16.config +++ b/second/configs/pointpillars/car/xyres_16.config @@ -103,15 +103,15 @@ model: { train_input_reader: { dataset: { dataset_class_name: "KittiDataset" - kitti_info_path: "/media/yy/My Passport/datasets/kitti/kitti_infos_train.pkl" - kitti_root_path: "/media/yy/My Passport/datasets/kitti" + kitti_info_path: "/host/ssd/quan/data/tmp/kitti3d_second/kitti_infos_train.pkl" + kitti_root_path: "/host/ssd/quan/data/tmp/kitti3d_second" } - - batch_size: 2 + + batch_size: 8 preprocess: { max_number_of_voxels: 12000 shuffle_points: true - num_workers: 2 + num_workers: 8 groundtruth_localization_noise_std: [0.25, 0.25, 0.25] groundtruth_rotation_uniform_noise: [-0.15707963267, 0.15707963267] global_rotation_uniform_noise: [-0.78539816, 0.78539816] @@ -128,7 +128,7 @@ train_input_reader: { random_flip_y: true remove_environment: false database_sampler { - database_info_path: "/media/yy/My Passport/datasets/kitti/kitti_dbinfos_train.pkl" + database_info_path: "/host/ssd/quan/data/tmp/kitti3d_second/kitti_dbinfos_train.pkl" sample_groups { name_to_max_num { key: "Car" @@ -182,12 +182,12 @@ train_config: { eval_input_reader: { dataset: { dataset_class_name: "KittiDataset" - kitti_info_path: "/media/yy/My Passport/datasets/kitti/kitti_infos_val.pkl" - # kitti_info_path: "/media/yy/My Passport/datasets/kitti/kitti_infos_test.pkl" - kitti_root_path: "/media/yy/My Passport/datasets/kitti" + kitti_info_path: "/host/ssd/quan/data/tmp/kitti3d_second/kitti_infos_val.pkl" + # kitti_info_path: "/host/ssd/quan/data/tmp/kitti3d_second/kitti_infos_test.pkl" + kitti_root_path: "/host/ssd/quan/data/tmp/kitti3d_second" } batch_size: 2 - + preprocess: { max_number_of_voxels: 12000 shuffle_points: false diff --git a/second/configs/pointpillars/car/xyres_20.config b/second/configs/pointpillars/car/xyres_20.config index 29627d96..64f66737 100644 --- a/second/configs/pointpillars/car/xyres_20.config +++ b/second/configs/pointpillars/car/xyres_20.config @@ -11,7 +11,7 @@ model: { module_class_name: "PillarFeatureNet" num_filters: [64] with_distance: false - num_input_features: 4 + num_input_features: 6 } middle_feature_extractor: { module_class_name: "PointPillarsScatter" @@ -46,7 +46,7 @@ model: { classification_weight: 1.0 localization_weight: 2.0 } - num_point_features: 4 # model's num point feature should be independent of dataset + num_point_features: 6 # model's num point feature should be independent of dataset # Outputs use_sigmoid_score: true encode_background_as_zeros: true @@ -104,10 +104,10 @@ model: { train_input_reader: { dataset: { dataset_class_name: "KittiDataset" - kitti_info_path: "/media/yy/960evo/datasets/kitti/kitti_infos_train.pkl" - kitti_root_path: "/media/yy/960evo/datasets/kitti" + kitti_info_path: "/host/ssd/quan/data/tmp/kitti3d_second/kitti_infos_train.pkl" + kitti_root_path: "/host/ssd/quan/data/tmp/kitti3d_second" } - + batch_size: 2 preprocess: { max_number_of_voxels: 12000 @@ -129,7 +129,7 @@ train_input_reader: { random_flip_y: true remove_environment: false database_sampler { - database_info_path: "/media/yy/960evo/datasets/kitti/kitti_dbinfos_train.pkl" + database_info_path: "/host/ssd/quan/data/tmp/kitti3d_second/kitti_dbinfos_train.pkl" sample_groups { name_to_max_num { key: "Car" @@ -183,12 +183,12 @@ train_config: { eval_input_reader: { dataset: { dataset_class_name: "KittiDataset" - kitti_info_path: "/media/yy/960evo/datasets/kitti/kitti_infos_val.pkl" - # kitti_info_path: "/media/yy/960evo/datasets/kitti/kitti_infos_test.pkl" - kitti_root_path: "/media/yy/960evo/datasets/kitti" + kitti_info_path: "/host/ssd/quan/data/tmp/kitti3d_second/kitti_infos_val.pkl" + # kitti_info_path: "/host/ssd/quan/data/tmp/kitti3d_second/kitti_infos_test.pkl" + kitti_root_path: "/host/ssd/quan/data/tmp/kitti3d_second" } batch_size: 2 - + preprocess: { max_number_of_voxels: 12000 shuffle_points: false diff --git a/second/configs/pointpillars/pp_pretrain.config b/second/configs/pointpillars/pp_pretrain.config index 4cabea66..c6ad0b3e 100644 --- a/second/configs/pointpillars/pp_pretrain.config +++ b/second/configs/pointpillars/pp_pretrain.config @@ -11,7 +11,7 @@ model: { module_class_name: "PillarFeatureNet" num_filters: [64] with_distance: false - num_input_features: 4 + num_input_features: 6 } middle_feature_extractor: { module_class_name: "PointPillarsScatter" @@ -46,7 +46,7 @@ model: { classification_weight: 1.0 localization_weight: 2.0 } - num_point_features: 4 # model's num point feature should be independent of dataset + num_point_features: 6 # model's num point feature should be independent of dataset # Outputs use_sigmoid_score: true encode_background_as_zeros: true @@ -148,11 +148,11 @@ model: { train_input_reader: { dataset: { dataset_class_name: "KittiDataset" - kitti_info_path: "/media/yy/software/datasets/kitti/kitti_infos_train.pkl" - kitti_root_path: "/media/yy/software/datasets/kitti" + kitti_info_path: "/host/ssd/quan/data/tmp/kitti3d_second/kitti_infos_train.pkl" + kitti_root_path: "/host/ssd/quan/data/tmp/kitti3d_second" } - - batch_size: 2 + + batch_size: 8 preprocess: { max_number_of_voxels: 12000 shuffle_points: true @@ -173,7 +173,7 @@ train_input_reader: { random_flip_y: true remove_environment: false database_sampler { - database_info_path: "/media/yy/software/datasets/kitti/kitti_dbinfos_train.pkl" + database_info_path: "/host/ssd/quan/data/tmp/kitti3d_second/kitti_dbinfos_train.pkl" sample_groups { name_to_max_num { key: "Car" @@ -241,12 +241,12 @@ train_config: { eval_input_reader: { dataset: { dataset_class_name: "KittiDataset" - kitti_info_path: "/media/yy/software/datasets/kitti/kitti_infos_val.pkl" - # kitti_info_path: "/media/yy/software/datasets/kitti/kitti_infos_test.pkl" - kitti_root_path: "/media/yy/software/datasets/kitti" + kitti_info_path: "/host/ssd/quan/data/tmp/kitti3d_second/kitti_infos_val.pkl" + # kitti_info_path: "/host/ssd/quan/data/tmp/kitti3d_second/kitti_infos_test.pkl" + kitti_root_path: "/host/ssd/quan/data/tmp/kitti3d_second" } batch_size: 2 - + preprocess: { max_number_of_voxels: 12000 shuffle_points: false diff --git a/second/create_data.py b/second/create_data.py index 7cb4e3e8..96606f95 100644 --- a/second/create_data.py +++ b/second/create_data.py @@ -8,9 +8,10 @@ import second.data.nuscenes_dataset as nu_ds from second.data.all_dataset import create_groundtruth_database -def kitti_data_prep(root_path): - kitti_ds.create_kitti_info_file(root_path) - kitti_ds.create_reduced_point_cloud(root_path) +def kitti_data_prep(root_path, use_disparity=False): + remove_outside = not use_disparity + kitti_ds.create_kitti_info_file(root_path, remove_outside=remove_outside) + kitti_ds.create_reduced_point_cloud(root_path, use_disparity=use_disparity) create_groundtruth_database("KittiDataset", root_path, Path(root_path) / "kitti_infos_train.pkl") def nuscenes_data_prep(root_path, version, dataset_name, max_sweeps=10): diff --git a/second/data/ImageSets/test.txt b/second/data/ImageSets/test_bak.txt similarity index 100% rename from second/data/ImageSets/test.txt rename to second/data/ImageSets/test_bak.txt diff --git a/second/data/ImageSets/test_debug.txt b/second/data/ImageSets/test_debug.txt new file mode 100644 index 00000000..b5b58f34 --- /dev/null +++ b/second/data/ImageSets/test_debug.txt @@ -0,0 +1 @@ +000001 \ No newline at end of file diff --git a/second/data/ImageSets/test_short.txt b/second/data/ImageSets/test_short.txt new file mode 100644 index 00000000..01c68aac --- /dev/null +++ b/second/data/ImageSets/test_short.txt @@ -0,0 +1,7480 @@ +000000 +000001 +000002 +000003 +000004 +000005 +000006 +000007 +000008 +000009 +000010 +000011 +000012 +000013 +000014 +000015 +000016 +000017 +000018 +000019 +000020 +000021 +000022 +000023 +000024 +000025 +000026 +000027 +000028 +000029 +000030 +000031 +000032 +000033 +000034 +000035 +000036 +000037 +000038 +000039 +000040 +000041 +000042 +000043 +000044 +000045 +000046 +000047 +000048 +000049 +000050 +000051 +000052 +000053 +000054 +000055 +000056 +000057 +000058 +000059 +000060 +000061 +000062 +000063 +000064 +000065 +000066 +000067 +000068 +000069 +000070 +000071 +000072 +000073 +000074 +000075 +000076 +000077 +000078 +000079 +000080 +000081 +000082 +000083 +000084 +000085 +000086 +000087 +000088 +000089 +000090 +000091 +000092 +000093 +000094 +000095 +000096 +000097 +000098 +000099 +000100 +000101 +000102 +000103 +000104 +000105 +000106 +000107 +000108 +000109 +000110 +000111 +000112 +000113 +000114 +000115 +000116 +000117 +000118 +000119 +000120 +000121 +000122 +000123 +000124 +000125 +000126 +000127 +000128 +000129 +000130 +000131 +000132 +000133 +000134 +000135 +000136 +000137 +000138 +000139 +000140 +000141 +000142 +000143 +000144 +000145 +000146 +000147 +000148 +000149 +000150 +000151 +000152 +000153 +000154 +000155 +000156 +000157 +000158 +000159 +000160 +000161 +000162 +000163 +000164 +000165 +000166 +000167 +000168 +000169 +000170 +000171 +000172 +000173 +000174 +000175 +000176 +000177 +000178 +000179 +000180 +000181 +000182 +000183 +000184 +000185 +000186 +000187 +000188 +000189 +000190 +000191 +000192 +000193 +000194 +000195 +000196 +000197 +000198 +000199 +000200 +000201 +000202 +000203 +000204 +000205 +000206 +000207 +000208 +000209 +000210 +000211 +000212 +000213 +000214 +000215 +000216 +000217 +000218 +000219 +000220 +000221 +000222 +000223 +000224 +000225 +000226 +000227 +000228 +000229 +000230 +000231 +000232 +000233 +000234 +000235 +000236 +000237 +000238 +000239 +000240 +000241 +000242 +000243 +000244 +000245 +000246 +000247 +000248 +000249 +000250 +000251 +000252 +000253 +000254 +000255 +000256 +000257 +000258 +000259 +000260 +000261 +000262 +000263 +000264 +000265 +000266 +000267 +000268 +000269 +000270 +000271 +000272 +000273 +000274 +000275 +000276 +000277 +000278 +000279 +000280 +000281 +000282 +000283 +000284 +000285 +000286 +000287 +000288 +000289 +000290 +000291 +000292 +000293 +000294 +000295 +000296 +000297 +000298 +000299 +000300 +000301 +000302 +000303 +000304 +000305 +000306 +000307 +000308 +000309 +000310 +000311 +000312 +000313 +000314 +000315 +000316 +000317 +000318 +000319 +000320 +000321 +000322 +000323 +000324 +000325 +000326 +000327 +000328 +000329 +000330 +000331 +000332 +000333 +000334 +000335 +000336 +000337 +000338 +000339 +000340 +000341 +000342 +000343 +000344 +000345 +000346 +000347 +000348 +000349 +000350 +000351 +000352 +000353 +000354 +000355 +000356 +000357 +000358 +000359 +000360 +000361 +000362 +000363 +000364 +000365 +000366 +000367 +000368 +000369 +000370 +000371 +000372 +000373 +000374 +000375 +000376 +000377 +000378 +000379 +000380 +000381 +000382 +000383 +000384 +000385 +000386 +000387 +000388 +000389 +000390 +000391 +000392 +000393 +000394 +000395 +000396 +000397 +000398 +000399 +000400 +000401 +000402 +000403 +000404 +000405 +000406 +000407 +000408 +000409 +000410 +000411 +000412 +000413 +000414 +000415 +000416 +000417 +000418 +000419 +000420 +000421 +000422 +000423 +000424 +000425 +000426 +000427 +000428 +000429 +000430 +000431 +000432 +000433 +000434 +000435 +000436 +000437 +000438 +000439 +000440 +000441 +000442 +000443 +000444 +000445 +000446 +000447 +000448 +000449 +000450 +000451 +000452 +000453 +000454 +000455 +000456 +000457 +000458 +000459 +000460 +000461 +000462 +000463 +000464 +000465 +000466 +000467 +000468 +000469 +000470 +000471 +000472 +000473 +000474 +000475 +000476 +000477 +000478 +000479 +000480 +000481 +000482 +000483 +000484 +000485 +000486 +000487 +000488 +000489 +000490 +000491 +000492 +000493 +000494 +000495 +000496 +000497 +000498 +000499 +000500 +000501 +000502 +000503 +000504 +000505 +000506 +000507 +000508 +000509 +000510 +000511 +000512 +000513 +000514 +000515 +000516 +000517 +000518 +000519 +000520 +000521 +000522 +000523 +000524 +000525 +000526 +000527 +000528 +000529 +000530 +000531 +000532 +000533 +000534 +000535 +000536 +000537 +000538 +000539 +000540 +000541 +000542 +000543 +000544 +000545 +000546 +000547 +000548 +000549 +000550 +000551 +000552 +000553 +000554 +000555 +000556 +000557 +000558 +000559 +000560 +000561 +000562 +000563 +000564 +000565 +000566 +000567 +000568 +000569 +000570 +000571 +000572 +000573 +000574 +000575 +000576 +000577 +000578 +000579 +000580 +000581 +000582 +000583 +000584 +000585 +000586 +000587 +000588 +000589 +000590 +000591 +000592 +000593 +000594 +000595 +000596 +000597 +000598 +000599 +000600 +000601 +000602 +000603 +000604 +000605 +000606 +000607 +000608 +000609 +000610 +000611 +000612 +000613 +000614 +000615 +000616 +000617 +000618 +000619 +000620 +000621 +000622 +000623 +000624 +000625 +000626 +000627 +000628 +000629 +000630 +000631 +000632 +000633 +000634 +000635 +000636 +000637 +000638 +000639 +000640 +000641 +000642 +000643 +000644 +000645 +000646 +000647 +000648 +000649 +000650 +000651 +000652 +000653 +000654 +000655 +000656 +000657 +000658 +000659 +000660 +000661 +000662 +000663 +000664 +000665 +000666 +000667 +000668 +000669 +000670 +000671 +000672 +000673 +000674 +000675 +000676 +000677 +000678 +000679 +000680 +000681 +000682 +000683 +000684 +000685 +000686 +000687 +000688 +000689 +000690 +000691 +000692 +000693 +000694 +000695 +000696 +000697 +000698 +000699 +000700 +000701 +000702 +000703 +000704 +000705 +000706 +000707 +000708 +000709 +000710 +000711 +000712 +000713 +000714 +000715 +000716 +000717 +000718 +000719 +000720 +000721 +000722 +000723 +000724 +000725 +000726 +000727 +000728 +000729 +000730 +000731 +000732 +000733 +000734 +000735 +000736 +000737 +000738 +000739 +000740 +000741 +000742 +000743 +000744 +000745 +000746 +000747 +000748 +000749 +000750 +000751 +000752 +000753 +000754 +000755 +000756 +000757 +000758 +000759 +000760 +000761 +000762 +000763 +000764 +000765 +000766 +000767 +000768 +000769 +000770 +000771 +000772 +000773 +000774 +000775 +000776 +000777 +000778 +000779 +000780 +000781 +000782 +000783 +000784 +000785 +000786 +000787 +000788 +000789 +000790 +000791 +000792 +000793 +000794 +000795 +000796 +000797 +000798 +000799 +000800 +000801 +000802 +000803 +000804 +000805 +000806 +000807 +000808 +000809 +000810 +000811 +000812 +000813 +000814 +000815 +000816 +000817 +000818 +000819 +000820 +000821 +000822 +000823 +000824 +000825 +000826 +000827 +000828 +000829 +000830 +000831 +000832 +000833 +000834 +000835 +000836 +000837 +000838 +000839 +000840 +000841 +000842 +000843 +000844 +000845 +000846 +000847 +000848 +000849 +000850 +000851 +000852 +000853 +000854 +000855 +000856 +000857 +000858 +000859 +000860 +000861 +000862 +000863 +000864 +000865 +000866 +000867 +000868 +000869 +000870 +000871 +000872 +000873 +000874 +000875 +000876 +000877 +000878 +000879 +000880 +000881 +000882 +000883 +000884 +000885 +000886 +000887 +000888 +000889 +000890 +000891 +000892 +000893 +000894 +000895 +000896 +000897 +000898 +000899 +000900 +000901 +000902 +000903 +000904 +000905 +000906 +000907 +000908 +000909 +000910 +000911 +000912 +000913 +000914 +000915 +000916 +000917 +000918 +000919 +000920 +000921 +000922 +000923 +000924 +000925 +000926 +000927 +000928 +000929 +000930 +000931 +000932 +000933 +000934 +000935 +000936 +000937 +000938 +000939 +000940 +000941 +000942 +000943 +000944 +000945 +000946 +000947 +000948 +000949 +000950 +000951 +000952 +000953 +000954 +000955 +000956 +000957 +000958 +000959 +000960 +000961 +000962 +000963 +000964 +000965 +000966 +000967 +000968 +000969 +000970 +000971 +000972 +000973 +000974 +000975 +000976 +000977 +000978 +000979 +000980 +000981 +000982 +000983 +000984 +000985 +000986 +000987 +000988 +000989 +000990 +000991 +000992 +000993 +000994 +000995 +000996 +000997 +000998 +000999 +001000 +001001 +001002 +001003 +001004 +001005 +001006 +001007 +001008 +001009 +001010 +001011 +001012 +001013 +001014 +001015 +001016 +001017 +001018 +001019 +001020 +001021 +001022 +001023 +001024 +001025 +001026 +001027 +001028 +001029 +001030 +001031 +001032 +001033 +001034 +001035 +001036 +001037 +001038 +001039 +001040 +001041 +001042 +001043 +001044 +001045 +001046 +001047 +001048 +001049 +001050 +001051 +001052 +001053 +001054 +001055 +001056 +001057 +001058 +001059 +001060 +001061 +001062 +001063 +001064 +001065 +001066 +001067 +001068 +001069 +001070 +001071 +001072 +001073 +001074 +001075 +001076 +001077 +001078 +001079 +001080 +001081 +001082 +001083 +001084 +001085 +001086 +001087 +001088 +001089 +001090 +001091 +001092 +001093 +001094 +001095 +001096 +001097 +001098 +001099 +001100 +001101 +001102 +001103 +001104 +001105 +001106 +001107 +001108 +001109 +001110 +001111 +001112 +001113 +001114 +001115 +001116 +001117 +001118 +001119 +001120 +001121 +001122 +001123 +001124 +001125 +001126 +001127 +001128 +001129 +001130 +001131 +001132 +001133 +001134 +001135 +001136 +001137 +001138 +001139 +001140 +001141 +001142 +001143 +001144 +001145 +001146 +001147 +001148 +001149 +001150 +001151 +001152 +001153 +001154 +001155 +001156 +001157 +001158 +001159 +001160 +001161 +001162 +001163 +001164 +001165 +001166 +001167 +001168 +001169 +001170 +001171 +001172 +001173 +001174 +001175 +001176 +001177 +001178 +001179 +001180 +001181 +001182 +001183 +001184 +001185 +001186 +001187 +001188 +001189 +001190 +001191 +001192 +001193 +001194 +001195 +001196 +001197 +001198 +001199 +001200 +001201 +001202 +001203 +001204 +001205 +001206 +001207 +001208 +001209 +001210 +001211 +001212 +001213 +001214 +001215 +001216 +001217 +001218 +001219 +001220 +001221 +001222 +001223 +001224 +001225 +001226 +001227 +001228 +001229 +001230 +001231 +001232 +001233 +001234 +001235 +001236 +001237 +001238 +001239 +001240 +001241 +001242 +001243 +001244 +001245 +001246 +001247 +001248 +001249 +001250 +001251 +001252 +001253 +001254 +001255 +001256 +001257 +001258 +001259 +001260 +001261 +001262 +001263 +001264 +001265 +001266 +001267 +001268 +001269 +001270 +001271 +001272 +001273 +001274 +001275 +001276 +001277 +001278 +001279 +001280 +001281 +001282 +001283 +001284 +001285 +001286 +001287 +001288 +001289 +001290 +001291 +001292 +001293 +001294 +001295 +001296 +001297 +001298 +001299 +001300 +001301 +001302 +001303 +001304 +001305 +001306 +001307 +001308 +001309 +001310 +001311 +001312 +001313 +001314 +001315 +001316 +001317 +001318 +001319 +001320 +001321 +001322 +001323 +001324 +001325 +001326 +001327 +001328 +001329 +001330 +001331 +001332 +001333 +001334 +001335 +001336 +001337 +001338 +001339 +001340 +001341 +001342 +001343 +001344 +001345 +001346 +001347 +001348 +001349 +001350 +001351 +001352 +001353 +001354 +001355 +001356 +001357 +001358 +001359 +001360 +001361 +001362 +001363 +001364 +001365 +001366 +001367 +001368 +001369 +001370 +001371 +001372 +001373 +001374 +001375 +001376 +001377 +001378 +001379 +001380 +001381 +001382 +001383 +001384 +001385 +001386 +001387 +001388 +001389 +001390 +001391 +001392 +001393 +001394 +001395 +001396 +001397 +001398 +001399 +001400 +001401 +001402 +001403 +001404 +001405 +001406 +001407 +001408 +001409 +001410 +001411 +001412 +001413 +001414 +001415 +001416 +001417 +001418 +001419 +001420 +001421 +001422 +001423 +001424 +001425 +001426 +001427 +001428 +001429 +001430 +001431 +001432 +001433 +001434 +001435 +001436 +001437 +001438 +001439 +001440 +001441 +001442 +001443 +001444 +001445 +001446 +001447 +001448 +001449 +001450 +001451 +001452 +001453 +001454 +001455 +001456 +001457 +001458 +001459 +001460 +001461 +001462 +001463 +001464 +001465 +001466 +001467 +001468 +001469 +001470 +001471 +001472 +001473 +001474 +001475 +001476 +001477 +001478 +001479 +001480 +001481 +001482 +001483 +001484 +001485 +001486 +001487 +001488 +001489 +001490 +001491 +001492 +001493 +001494 +001495 +001496 +001497 +001498 +001499 +001500 +001501 +001502 +001503 +001504 +001505 +001506 +001507 +001508 +001509 +001510 +001511 +001512 +001513 +001514 +001515 +001516 +001517 +001518 +001519 +001520 +001521 +001522 +001523 +001524 +001525 +001526 +001527 +001528 +001529 +001530 +001531 +001532 +001533 +001534 +001535 +001536 +001537 +001538 +001539 +001540 +001541 +001542 +001543 +001544 +001545 +001546 +001547 +001548 +001549 +001550 +001551 +001552 +001553 +001554 +001555 +001556 +001557 +001558 +001559 +001560 +001561 +001562 +001563 +001564 +001565 +001566 +001567 +001568 +001569 +001570 +001571 +001572 +001573 +001574 +001575 +001576 +001577 +001578 +001579 +001580 +001581 +001582 +001583 +001584 +001585 +001586 +001587 +001588 +001589 +001590 +001591 +001592 +001593 +001594 +001595 +001596 +001597 +001598 +001599 +001600 +001601 +001602 +001603 +001604 +001605 +001606 +001607 +001608 +001609 +001610 +001611 +001612 +001613 +001614 +001615 +001616 +001617 +001618 +001619 +001620 +001621 +001622 +001623 +001624 +001625 +001626 +001627 +001628 +001629 +001630 +001631 +001632 +001633 +001634 +001635 +001636 +001637 +001638 +001639 +001640 +001641 +001642 +001643 +001644 +001645 +001646 +001647 +001648 +001649 +001650 +001651 +001652 +001653 +001654 +001655 +001656 +001657 +001658 +001659 +001660 +001661 +001662 +001663 +001664 +001665 +001666 +001667 +001668 +001669 +001670 +001671 +001672 +001673 +001674 +001675 +001676 +001677 +001678 +001679 +001680 +001681 +001682 +001683 +001684 +001685 +001686 +001687 +001688 +001689 +001690 +001691 +001692 +001693 +001694 +001695 +001696 +001697 +001698 +001699 +001700 +001701 +001702 +001703 +001704 +001705 +001706 +001707 +001708 +001709 +001710 +001711 +001712 +001713 +001714 +001715 +001716 +001717 +001718 +001719 +001720 +001721 +001722 +001723 +001724 +001725 +001726 +001727 +001728 +001729 +001730 +001731 +001732 +001733 +001734 +001735 +001736 +001737 +001738 +001739 +001740 +001741 +001742 +001743 +001744 +001745 +001746 +001747 +001748 +001749 +001750 +001751 +001752 +001753 +001754 +001755 +001756 +001757 +001758 +001759 +001760 +001761 +001762 +001763 +001764 +001765 +001766 +001767 +001768 +001769 +001770 +001771 +001772 +001773 +001774 +001775 +001776 +001777 +001778 +001779 +001780 +001781 +001782 +001783 +001784 +001785 +001786 +001787 +001788 +001789 +001790 +001791 +001792 +001793 +001794 +001795 +001796 +001797 +001798 +001799 +001800 +001801 +001802 +001803 +001804 +001805 +001806 +001807 +001808 +001809 +001810 +001811 +001812 +001813 +001814 +001815 +001816 +001817 +001818 +001819 +001820 +001821 +001822 +001823 +001824 +001825 +001826 +001827 +001828 +001829 +001830 +001831 +001832 +001833 +001834 +001835 +001836 +001837 +001838 +001839 +001840 +001841 +001842 +001843 +001844 +001845 +001846 +001847 +001848 +001849 +001850 +001851 +001852 +001853 +001854 +001855 +001856 +001857 +001858 +001859 +001860 +001861 +001862 +001863 +001864 +001865 +001866 +001867 +001868 +001869 +001870 +001871 +001872 +001873 +001874 +001875 +001876 +001877 +001878 +001879 +001880 +001881 +001882 +001883 +001884 +001885 +001886 +001887 +001888 +001889 +001890 +001891 +001892 +001893 +001894 +001895 +001896 +001897 +001898 +001899 +001900 +001901 +001902 +001903 +001904 +001905 +001906 +001907 +001908 +001909 +001910 +001911 +001912 +001913 +001914 +001915 +001916 +001917 +001918 +001919 +001920 +001921 +001922 +001923 +001924 +001925 +001926 +001927 +001928 +001929 +001930 +001931 +001932 +001933 +001934 +001935 +001936 +001937 +001938 +001939 +001940 +001941 +001942 +001943 +001944 +001945 +001946 +001947 +001948 +001949 +001950 +001951 +001952 +001953 +001954 +001955 +001956 +001957 +001958 +001959 +001960 +001961 +001962 +001963 +001964 +001965 +001966 +001967 +001968 +001969 +001970 +001971 +001972 +001973 +001974 +001975 +001976 +001977 +001978 +001979 +001980 +001981 +001982 +001983 +001984 +001985 +001986 +001987 +001988 +001989 +001990 +001991 +001992 +001993 +001994 +001995 +001996 +001997 +001998 +001999 +002000 +002001 +002002 +002003 +002004 +002005 +002006 +002007 +002008 +002009 +002010 +002011 +002012 +002013 +002014 +002015 +002016 +002017 +002018 +002019 +002020 +002021 +002022 +002023 +002024 +002025 +002026 +002027 +002028 +002029 +002030 +002031 +002032 +002033 +002034 +002035 +002036 +002037 +002038 +002039 +002040 +002041 +002042 +002043 +002044 +002045 +002046 +002047 +002048 +002049 +002050 +002051 +002052 +002053 +002054 +002055 +002056 +002057 +002058 +002059 +002060 +002061 +002062 +002063 +002064 +002065 +002066 +002067 +002068 +002069 +002070 +002071 +002072 +002073 +002074 +002075 +002076 +002077 +002078 +002079 +002080 +002081 +002082 +002083 +002084 +002085 +002086 +002087 +002088 +002089 +002090 +002091 +002092 +002093 +002094 +002095 +002096 +002097 +002098 +002099 +002100 +002101 +002102 +002103 +002104 +002105 +002106 +002107 +002108 +002109 +002110 +002111 +002112 +002113 +002114 +002115 +002116 +002117 +002118 +002119 +002120 +002121 +002122 +002123 +002124 +002125 +002126 +002127 +002128 +002129 +002130 +002131 +002132 +002133 +002134 +002135 +002136 +002137 +002138 +002139 +002140 +002141 +002142 +002143 +002144 +002145 +002146 +002147 +002148 +002149 +002150 +002151 +002152 +002153 +002154 +002155 +002156 +002157 +002158 +002159 +002160 +002161 +002162 +002163 +002164 +002165 +002166 +002167 +002168 +002169 +002170 +002171 +002172 +002173 +002174 +002175 +002176 +002177 +002178 +002179 +002180 +002181 +002182 +002183 +002184 +002185 +002186 +002187 +002188 +002189 +002190 +002191 +002192 +002193 +002194 +002195 +002196 +002197 +002198 +002199 +002200 +002201 +002202 +002203 +002204 +002205 +002206 +002207 +002208 +002209 +002210 +002211 +002212 +002213 +002214 +002215 +002216 +002217 +002218 +002219 +002220 +002221 +002222 +002223 +002224 +002225 +002226 +002227 +002228 +002229 +002230 +002231 +002232 +002233 +002234 +002235 +002236 +002237 +002238 +002239 +002240 +002241 +002242 +002243 +002244 +002245 +002246 +002247 +002248 +002249 +002250 +002251 +002252 +002253 +002254 +002255 +002256 +002257 +002258 +002259 +002260 +002261 +002262 +002263 +002264 +002265 +002266 +002267 +002268 +002269 +002270 +002271 +002272 +002273 +002274 +002275 +002276 +002277 +002278 +002279 +002280 +002281 +002282 +002283 +002284 +002285 +002286 +002287 +002288 +002289 +002290 +002291 +002292 +002293 +002294 +002295 +002296 +002297 +002298 +002299 +002300 +002301 +002302 +002303 +002304 +002305 +002306 +002307 +002308 +002309 +002310 +002311 +002312 +002313 +002314 +002315 +002316 +002317 +002318 +002319 +002320 +002321 +002322 +002323 +002324 +002325 +002326 +002327 +002328 +002329 +002330 +002331 +002332 +002333 +002334 +002335 +002336 +002337 +002338 +002339 +002340 +002341 +002342 +002343 +002344 +002345 +002346 +002347 +002348 +002349 +002350 +002351 +002352 +002353 +002354 +002355 +002356 +002357 +002358 +002359 +002360 +002361 +002362 +002363 +002364 +002365 +002366 +002367 +002368 +002369 +002370 +002371 +002372 +002373 +002374 +002375 +002376 +002377 +002378 +002379 +002380 +002381 +002382 +002383 +002384 +002385 +002386 +002387 +002388 +002389 +002390 +002391 +002392 +002393 +002394 +002395 +002396 +002397 +002398 +002399 +002400 +002401 +002402 +002403 +002404 +002405 +002406 +002407 +002408 +002409 +002410 +002411 +002412 +002413 +002414 +002415 +002416 +002417 +002418 +002419 +002420 +002421 +002422 +002423 +002424 +002425 +002426 +002427 +002428 +002429 +002430 +002431 +002432 +002433 +002434 +002435 +002436 +002437 +002438 +002439 +002440 +002441 +002442 +002443 +002444 +002445 +002446 +002447 +002448 +002449 +002450 +002451 +002452 +002453 +002454 +002455 +002456 +002457 +002458 +002459 +002460 +002461 +002462 +002463 +002464 +002465 +002466 +002467 +002468 +002469 +002470 +002471 +002472 +002473 +002474 +002475 +002476 +002477 +002478 +002479 +002480 +002481 +002482 +002483 +002484 +002485 +002486 +002487 +002488 +002489 +002490 +002491 +002492 +002493 +002494 +002495 +002496 +002497 +002498 +002499 +002500 +002501 +002502 +002503 +002504 +002505 +002506 +002507 +002508 +002509 +002510 +002511 +002512 +002513 +002514 +002515 +002516 +002517 +002518 +002519 +002520 +002521 +002522 +002523 +002524 +002525 +002526 +002527 +002528 +002529 +002530 +002531 +002532 +002533 +002534 +002535 +002536 +002537 +002538 +002539 +002540 +002541 +002542 +002543 +002544 +002545 +002546 +002547 +002548 +002549 +002550 +002551 +002552 +002553 +002554 +002555 +002556 +002557 +002558 +002559 +002560 +002561 +002562 +002563 +002564 +002565 +002566 +002567 +002568 +002569 +002570 +002571 +002572 +002573 +002574 +002575 +002576 +002577 +002578 +002579 +002580 +002581 +002582 +002583 +002584 +002585 +002586 +002587 +002588 +002589 +002590 +002591 +002592 +002593 +002594 +002595 +002596 +002597 +002598 +002599 +002600 +002601 +002602 +002603 +002604 +002605 +002606 +002607 +002608 +002609 +002610 +002611 +002612 +002613 +002614 +002615 +002616 +002617 +002618 +002619 +002620 +002621 +002622 +002623 +002624 +002625 +002626 +002627 +002628 +002629 +002630 +002631 +002632 +002633 +002634 +002635 +002636 +002637 +002638 +002639 +002640 +002641 +002642 +002643 +002644 +002645 +002646 +002647 +002648 +002649 +002650 +002651 +002652 +002653 +002654 +002655 +002656 +002657 +002658 +002659 +002660 +002661 +002662 +002663 +002664 +002665 +002666 +002667 +002668 +002669 +002670 +002671 +002672 +002673 +002674 +002675 +002676 +002677 +002678 +002679 +002680 +002681 +002682 +002683 +002684 +002685 +002686 +002687 +002688 +002689 +002690 +002691 +002692 +002693 +002694 +002695 +002696 +002697 +002698 +002699 +002700 +002701 +002702 +002703 +002704 +002705 +002706 +002707 +002708 +002709 +002710 +002711 +002712 +002713 +002714 +002715 +002716 +002717 +002718 +002719 +002720 +002721 +002722 +002723 +002724 +002725 +002726 +002727 +002728 +002729 +002730 +002731 +002732 +002733 +002734 +002735 +002736 +002737 +002738 +002739 +002740 +002741 +002742 +002743 +002744 +002745 +002746 +002747 +002748 +002749 +002750 +002751 +002752 +002753 +002754 +002755 +002756 +002757 +002758 +002759 +002760 +002761 +002762 +002763 +002764 +002765 +002766 +002767 +002768 +002769 +002770 +002771 +002772 +002773 +002774 +002775 +002776 +002777 +002778 +002779 +002780 +002781 +002782 +002783 +002784 +002785 +002786 +002787 +002788 +002789 +002790 +002791 +002792 +002793 +002794 +002795 +002796 +002797 +002798 +002799 +002800 +002801 +002802 +002803 +002804 +002805 +002806 +002807 +002808 +002809 +002810 +002811 +002812 +002813 +002814 +002815 +002816 +002817 +002818 +002819 +002820 +002821 +002822 +002823 +002824 +002825 +002826 +002827 +002828 +002829 +002830 +002831 +002832 +002833 +002834 +002835 +002836 +002837 +002838 +002839 +002840 +002841 +002842 +002843 +002844 +002845 +002846 +002847 +002848 +002849 +002850 +002851 +002852 +002853 +002854 +002855 +002856 +002857 +002858 +002859 +002860 +002861 +002862 +002863 +002864 +002865 +002866 +002867 +002868 +002869 +002870 +002871 +002872 +002873 +002874 +002875 +002876 +002877 +002878 +002879 +002880 +002881 +002882 +002883 +002884 +002885 +002886 +002887 +002888 +002889 +002890 +002891 +002892 +002893 +002894 +002895 +002896 +002897 +002898 +002899 +002900 +002901 +002902 +002903 +002904 +002905 +002906 +002907 +002908 +002909 +002910 +002911 +002912 +002913 +002914 +002915 +002916 +002917 +002918 +002919 +002920 +002921 +002922 +002923 +002924 +002925 +002926 +002927 +002928 +002929 +002930 +002931 +002932 +002933 +002934 +002935 +002936 +002937 +002938 +002939 +002940 +002941 +002942 +002943 +002944 +002945 +002946 +002947 +002948 +002949 +002950 +002951 +002952 +002953 +002954 +002955 +002956 +002957 +002958 +002959 +002960 +002961 +002962 +002963 +002964 +002965 +002966 +002967 +002968 +002969 +002970 +002971 +002972 +002973 +002974 +002975 +002976 +002977 +002978 +002979 +002980 +002981 +002982 +002983 +002984 +002985 +002986 +002987 +002988 +002989 +002990 +002991 +002992 +002993 +002994 +002995 +002996 +002997 +002998 +002999 +003000 +003001 +003002 +003003 +003004 +003005 +003006 +003007 +003008 +003009 +003010 +003011 +003012 +003013 +003014 +003015 +003016 +003017 +003018 +003019 +003020 +003021 +003022 +003023 +003024 +003025 +003026 +003027 +003028 +003029 +003030 +003031 +003032 +003033 +003034 +003035 +003036 +003037 +003038 +003039 +003040 +003041 +003042 +003043 +003044 +003045 +003046 +003047 +003048 +003049 +003050 +003051 +003052 +003053 +003054 +003055 +003056 +003057 +003058 +003059 +003060 +003061 +003062 +003063 +003064 +003065 +003066 +003067 +003068 +003069 +003070 +003071 +003072 +003073 +003074 +003075 +003076 +003077 +003078 +003079 +003080 +003081 +003082 +003083 +003084 +003085 +003086 +003087 +003088 +003089 +003090 +003091 +003092 +003093 +003094 +003095 +003096 +003097 +003098 +003099 +003100 +003101 +003102 +003103 +003104 +003105 +003106 +003107 +003108 +003109 +003110 +003111 +003112 +003113 +003114 +003115 +003116 +003117 +003118 +003119 +003120 +003121 +003122 +003123 +003124 +003125 +003126 +003127 +003128 +003129 +003130 +003131 +003132 +003133 +003134 +003135 +003136 +003137 +003138 +003139 +003140 +003141 +003142 +003143 +003144 +003145 +003146 +003147 +003148 +003149 +003150 +003151 +003152 +003153 +003154 +003155 +003156 +003157 +003158 +003159 +003160 +003161 +003162 +003163 +003164 +003165 +003166 +003167 +003168 +003169 +003170 +003171 +003172 +003173 +003174 +003175 +003176 +003177 +003178 +003179 +003180 +003181 +003182 +003183 +003184 +003185 +003186 +003187 +003188 +003189 +003190 +003191 +003192 +003193 +003194 +003195 +003196 +003197 +003198 +003199 +003200 +003201 +003202 +003203 +003204 +003205 +003206 +003207 +003208 +003209 +003210 +003211 +003212 +003213 +003214 +003215 +003216 +003217 +003218 +003219 +003220 +003221 +003222 +003223 +003224 +003225 +003226 +003227 +003228 +003229 +003230 +003231 +003232 +003233 +003234 +003235 +003236 +003237 +003238 +003239 +003240 +003241 +003242 +003243 +003244 +003245 +003246 +003247 +003248 +003249 +003250 +003251 +003252 +003253 +003254 +003255 +003256 +003257 +003258 +003259 +003260 +003261 +003262 +003263 +003264 +003265 +003266 +003267 +003268 +003269 +003270 +003271 +003272 +003273 +003274 +003275 +003276 +003277 +003278 +003279 +003280 +003281 +003282 +003283 +003284 +003285 +003286 +003287 +003288 +003289 +003290 +003291 +003292 +003293 +003294 +003295 +003296 +003297 +003298 +003299 +003300 +003301 +003302 +003303 +003304 +003305 +003306 +003307 +003308 +003309 +003310 +003311 +003312 +003313 +003314 +003315 +003316 +003317 +003318 +003319 +003320 +003321 +003322 +003323 +003324 +003325 +003326 +003327 +003328 +003329 +003330 +003331 +003332 +003333 +003334 +003335 +003336 +003337 +003338 +003339 +003340 +003341 +003342 +003343 +003344 +003345 +003346 +003347 +003348 +003349 +003350 +003351 +003352 +003353 +003354 +003355 +003356 +003357 +003358 +003359 +003360 +003361 +003362 +003363 +003364 +003365 +003366 +003367 +003368 +003369 +003370 +003371 +003372 +003373 +003374 +003375 +003376 +003377 +003378 +003379 +003380 +003381 +003382 +003383 +003384 +003385 +003386 +003387 +003388 +003389 +003390 +003391 +003392 +003393 +003394 +003395 +003396 +003397 +003398 +003399 +003400 +003401 +003402 +003403 +003404 +003405 +003406 +003407 +003408 +003409 +003410 +003411 +003412 +003413 +003414 +003415 +003416 +003417 +003418 +003419 +003420 +003421 +003422 +003423 +003424 +003425 +003426 +003427 +003428 +003429 +003430 +003431 +003432 +003433 +003434 +003435 +003436 +003437 +003438 +003439 +003440 +003441 +003442 +003443 +003444 +003445 +003446 +003447 +003448 +003449 +003450 +003451 +003452 +003453 +003454 +003455 +003456 +003457 +003458 +003459 +003460 +003461 +003462 +003463 +003464 +003465 +003466 +003467 +003468 +003469 +003470 +003471 +003472 +003473 +003474 +003475 +003476 +003477 +003478 +003479 +003480 +003481 +003482 +003483 +003484 +003485 +003486 +003487 +003488 +003489 +003490 +003491 +003492 +003493 +003494 +003495 +003496 +003497 +003498 +003499 +003500 +003501 +003502 +003503 +003504 +003505 +003506 +003507 +003508 +003509 +003510 +003511 +003512 +003513 +003514 +003515 +003516 +003517 +003518 +003519 +003520 +003521 +003522 +003523 +003524 +003525 +003526 +003527 +003528 +003529 +003530 +003531 +003532 +003533 +003534 +003535 +003536 +003537 +003538 +003539 +003540 +003541 +003542 +003543 +003544 +003545 +003546 +003547 +003548 +003549 +003550 +003551 +003552 +003553 +003554 +003555 +003556 +003557 +003558 +003559 +003560 +003561 +003562 +003563 +003564 +003565 +003566 +003567 +003568 +003569 +003570 +003571 +003572 +003573 +003574 +003575 +003576 +003577 +003578 +003579 +003580 +003581 +003582 +003583 +003584 +003585 +003586 +003587 +003588 +003589 +003590 +003591 +003592 +003593 +003594 +003595 +003596 +003597 +003598 +003599 +003600 +003601 +003602 +003603 +003604 +003605 +003606 +003607 +003608 +003609 +003610 +003611 +003612 +003613 +003614 +003615 +003616 +003617 +003618 +003619 +003620 +003621 +003622 +003623 +003624 +003625 +003626 +003627 +003628 +003629 +003630 +003631 +003632 +003633 +003634 +003635 +003636 +003637 +003638 +003639 +003640 +003641 +003642 +003643 +003644 +003645 +003646 +003647 +003648 +003649 +003650 +003651 +003652 +003653 +003654 +003655 +003656 +003657 +003658 +003659 +003660 +003661 +003662 +003663 +003664 +003665 +003666 +003667 +003668 +003669 +003670 +003671 +003672 +003673 +003674 +003675 +003676 +003677 +003678 +003679 +003680 +003681 +003682 +003683 +003684 +003685 +003686 +003687 +003688 +003689 +003690 +003691 +003692 +003693 +003694 +003695 +003696 +003697 +003698 +003699 +003700 +003701 +003702 +003703 +003704 +003705 +003706 +003707 +003708 +003709 +003710 +003711 +003712 +003713 +003714 +003715 +003716 +003717 +003718 +003719 +003720 +003721 +003722 +003723 +003724 +003725 +003726 +003727 +003728 +003729 +003730 +003731 +003732 +003733 +003734 +003735 +003736 +003737 +003738 +003739 +003740 +003741 +003742 +003743 +003744 +003745 +003746 +003747 +003748 +003749 +003750 +003751 +003752 +003753 +003754 +003755 +003756 +003757 +003758 +003759 +003760 +003761 +003762 +003763 +003764 +003765 +003766 +003767 +003768 +003769 +003770 +003771 +003772 +003773 +003774 +003775 +003776 +003777 +003778 +003779 +003780 +003781 +003782 +003783 +003784 +003785 +003786 +003787 +003788 +003789 +003790 +003791 +003792 +003793 +003794 +003795 +003796 +003797 +003798 +003799 +003800 +003801 +003802 +003803 +003804 +003805 +003806 +003807 +003808 +003809 +003810 +003811 +003812 +003813 +003814 +003815 +003816 +003817 +003818 +003819 +003820 +003821 +003822 +003823 +003824 +003825 +003826 +003827 +003828 +003829 +003830 +003831 +003832 +003833 +003834 +003835 +003836 +003837 +003838 +003839 +003840 +003841 +003842 +003843 +003844 +003845 +003846 +003847 +003848 +003849 +003850 +003851 +003852 +003853 +003854 +003855 +003856 +003857 +003858 +003859 +003860 +003861 +003862 +003863 +003864 +003865 +003866 +003867 +003868 +003869 +003870 +003871 +003872 +003873 +003874 +003875 +003876 +003877 +003878 +003879 +003880 +003881 +003882 +003883 +003884 +003885 +003886 +003887 +003888 +003889 +003890 +003891 +003892 +003893 +003894 +003895 +003896 +003897 +003898 +003899 +003900 +003901 +003902 +003903 +003904 +003905 +003906 +003907 +003908 +003909 +003910 +003911 +003912 +003913 +003914 +003915 +003916 +003917 +003918 +003919 +003920 +003921 +003922 +003923 +003924 +003925 +003926 +003927 +003928 +003929 +003930 +003931 +003932 +003933 +003934 +003935 +003936 +003937 +003938 +003939 +003940 +003941 +003942 +003943 +003944 +003945 +003946 +003947 +003948 +003949 +003950 +003951 +003952 +003953 +003954 +003955 +003956 +003957 +003958 +003959 +003960 +003961 +003962 +003963 +003964 +003965 +003966 +003967 +003968 +003969 +003970 +003971 +003972 +003973 +003974 +003975 +003976 +003977 +003978 +003979 +003980 +003981 +003982 +003983 +003984 +003985 +003986 +003987 +003988 +003989 +003990 +003991 +003992 +003993 +003994 +003995 +003996 +003997 +003998 +003999 +004000 +004001 +004002 +004003 +004004 +004005 +004006 +004007 +004008 +004009 +004010 +004011 +004012 +004013 +004014 +004015 +004016 +004017 +004018 +004019 +004020 +004021 +004022 +004023 +004024 +004025 +004026 +004027 +004028 +004029 +004030 +004031 +004032 +004033 +004034 +004035 +004036 +004037 +004038 +004039 +004040 +004041 +004042 +004043 +004044 +004045 +004046 +004047 +004048 +004049 +004050 +004051 +004052 +004053 +004054 +004055 +004056 +004057 +004058 +004059 +004060 +004061 +004062 +004063 +004064 +004065 +004066 +004067 +004068 +004069 +004070 +004071 +004072 +004073 +004074 +004075 +004076 +004077 +004078 +004079 +004080 +004081 +004082 +004083 +004084 +004085 +004086 +004087 +004088 +004089 +004090 +004091 +004092 +004093 +004094 +004095 +004096 +004097 +004098 +004099 +004100 +004101 +004102 +004103 +004104 +004105 +004106 +004107 +004108 +004109 +004110 +004111 +004112 +004113 +004114 +004115 +004116 +004117 +004118 +004119 +004120 +004121 +004122 +004123 +004124 +004125 +004126 +004127 +004128 +004129 +004130 +004131 +004132 +004133 +004134 +004135 +004136 +004137 +004138 +004139 +004140 +004141 +004142 +004143 +004144 +004145 +004146 +004147 +004148 +004149 +004150 +004151 +004152 +004153 +004154 +004155 +004156 +004157 +004158 +004159 +004160 +004161 +004162 +004163 +004164 +004165 +004166 +004167 +004168 +004169 +004170 +004171 +004172 +004173 +004174 +004175 +004176 +004177 +004178 +004179 +004180 +004181 +004182 +004183 +004184 +004185 +004186 +004187 +004188 +004189 +004190 +004191 +004192 +004193 +004194 +004195 +004196 +004197 +004198 +004199 +004200 +004201 +004202 +004203 +004204 +004205 +004206 +004207 +004208 +004209 +004210 +004211 +004212 +004213 +004214 +004215 +004216 +004217 +004218 +004219 +004220 +004221 +004222 +004223 +004224 +004225 +004226 +004227 +004228 +004229 +004230 +004231 +004232 +004233 +004234 +004235 +004236 +004237 +004238 +004239 +004240 +004241 +004242 +004243 +004244 +004245 +004246 +004247 +004248 +004249 +004250 +004251 +004252 +004253 +004254 +004255 +004256 +004257 +004258 +004259 +004260 +004261 +004262 +004263 +004264 +004265 +004266 +004267 +004268 +004269 +004270 +004271 +004272 +004273 +004274 +004275 +004276 +004277 +004278 +004279 +004280 +004281 +004282 +004283 +004284 +004285 +004286 +004287 +004288 +004289 +004290 +004291 +004292 +004293 +004294 +004295 +004296 +004297 +004298 +004299 +004300 +004301 +004302 +004303 +004304 +004305 +004306 +004307 +004308 +004309 +004310 +004311 +004312 +004313 +004314 +004315 +004316 +004317 +004318 +004319 +004320 +004321 +004322 +004323 +004324 +004325 +004326 +004327 +004328 +004329 +004330 +004331 +004332 +004333 +004334 +004335 +004336 +004337 +004338 +004339 +004340 +004341 +004342 +004343 +004344 +004345 +004346 +004347 +004348 +004349 +004350 +004351 +004352 +004353 +004354 +004355 +004356 +004357 +004358 +004359 +004360 +004361 +004362 +004363 +004364 +004365 +004366 +004367 +004368 +004369 +004370 +004371 +004372 +004373 +004374 +004375 +004376 +004377 +004378 +004379 +004380 +004381 +004382 +004383 +004384 +004385 +004386 +004387 +004388 +004389 +004390 +004391 +004392 +004393 +004394 +004395 +004396 +004397 +004398 +004399 +004400 +004401 +004402 +004403 +004404 +004405 +004406 +004407 +004408 +004409 +004410 +004411 +004412 +004413 +004414 +004415 +004416 +004417 +004418 +004419 +004420 +004421 +004422 +004423 +004424 +004425 +004426 +004427 +004428 +004429 +004430 +004431 +004432 +004433 +004434 +004435 +004436 +004437 +004438 +004439 +004440 +004441 +004442 +004443 +004444 +004445 +004446 +004447 +004448 +004449 +004450 +004451 +004452 +004453 +004454 +004455 +004456 +004457 +004458 +004459 +004460 +004461 +004462 +004463 +004464 +004465 +004466 +004467 +004468 +004469 +004470 +004471 +004472 +004473 +004474 +004475 +004476 +004477 +004478 +004479 +004480 +004481 +004482 +004483 +004484 +004485 +004486 +004487 +004488 +004489 +004490 +004491 +004492 +004493 +004494 +004495 +004496 +004497 +004498 +004499 +004500 +004501 +004502 +004503 +004504 +004505 +004506 +004507 +004508 +004509 +004510 +004511 +004512 +004513 +004514 +004515 +004516 +004517 +004518 +004519 +004520 +004521 +004522 +004523 +004524 +004525 +004526 +004527 +004528 +004529 +004530 +004531 +004532 +004533 +004534 +004535 +004536 +004537 +004538 +004539 +004540 +004541 +004542 +004543 +004544 +004545 +004546 +004547 +004548 +004549 +004550 +004551 +004552 +004553 +004554 +004555 +004556 +004557 +004558 +004559 +004560 +004561 +004562 +004563 +004564 +004565 +004566 +004567 +004568 +004569 +004570 +004571 +004572 +004573 +004574 +004575 +004576 +004577 +004578 +004579 +004580 +004581 +004582 +004583 +004584 +004585 +004586 +004587 +004588 +004589 +004590 +004591 +004592 +004593 +004594 +004595 +004596 +004597 +004598 +004599 +004600 +004601 +004602 +004603 +004604 +004605 +004606 +004607 +004608 +004609 +004610 +004611 +004612 +004613 +004614 +004615 +004616 +004617 +004618 +004619 +004620 +004621 +004622 +004623 +004624 +004625 +004626 +004627 +004628 +004629 +004630 +004631 +004632 +004633 +004634 +004635 +004636 +004637 +004638 +004639 +004640 +004641 +004642 +004643 +004644 +004645 +004646 +004647 +004648 +004649 +004650 +004651 +004652 +004653 +004654 +004655 +004656 +004657 +004658 +004659 +004660 +004661 +004662 +004663 +004664 +004665 +004666 +004667 +004668 +004669 +004670 +004671 +004672 +004673 +004674 +004675 +004676 +004677 +004678 +004679 +004680 +004681 +004682 +004683 +004684 +004685 +004686 +004687 +004688 +004689 +004690 +004691 +004692 +004693 +004694 +004695 +004696 +004697 +004698 +004699 +004700 +004701 +004702 +004703 +004704 +004705 +004706 +004707 +004708 +004709 +004710 +004711 +004712 +004713 +004714 +004715 +004716 +004717 +004718 +004719 +004720 +004721 +004722 +004723 +004724 +004725 +004726 +004727 +004728 +004729 +004730 +004731 +004732 +004733 +004734 +004735 +004736 +004737 +004738 +004739 +004740 +004741 +004742 +004743 +004744 +004745 +004746 +004747 +004748 +004749 +004750 +004751 +004752 +004753 +004754 +004755 +004756 +004757 +004758 +004759 +004760 +004761 +004762 +004763 +004764 +004765 +004766 +004767 +004768 +004769 +004770 +004771 +004772 +004773 +004774 +004775 +004776 +004777 +004778 +004779 +004780 +004781 +004782 +004783 +004784 +004785 +004786 +004787 +004788 +004789 +004790 +004791 +004792 +004793 +004794 +004795 +004796 +004797 +004798 +004799 +004800 +004801 +004802 +004803 +004804 +004805 +004806 +004807 +004808 +004809 +004810 +004811 +004812 +004813 +004814 +004815 +004816 +004817 +004818 +004819 +004820 +004821 +004822 +004823 +004824 +004825 +004826 +004827 +004828 +004829 +004830 +004831 +004832 +004833 +004834 +004835 +004836 +004837 +004838 +004839 +004840 +004841 +004842 +004843 +004844 +004845 +004846 +004847 +004848 +004849 +004850 +004851 +004852 +004853 +004854 +004855 +004856 +004857 +004858 +004859 +004860 +004861 +004862 +004863 +004864 +004865 +004866 +004867 +004868 +004869 +004870 +004871 +004872 +004873 +004874 +004875 +004876 +004877 +004878 +004879 +004880 +004881 +004882 +004883 +004884 +004885 +004886 +004887 +004888 +004889 +004890 +004891 +004892 +004893 +004894 +004895 +004896 +004897 +004898 +004899 +004900 +004901 +004902 +004903 +004904 +004905 +004906 +004907 +004908 +004909 +004910 +004911 +004912 +004913 +004914 +004915 +004916 +004917 +004918 +004919 +004920 +004921 +004922 +004923 +004924 +004925 +004926 +004927 +004928 +004929 +004930 +004931 +004932 +004933 +004934 +004935 +004936 +004937 +004938 +004939 +004940 +004941 +004942 +004943 +004944 +004945 +004946 +004947 +004948 +004949 +004950 +004951 +004952 +004953 +004954 +004955 +004956 +004957 +004958 +004959 +004960 +004961 +004962 +004963 +004964 +004965 +004966 +004967 +004968 +004969 +004970 +004971 +004972 +004973 +004974 +004975 +004976 +004977 +004978 +004979 +004980 +004981 +004982 +004983 +004984 +004985 +004986 +004987 +004988 +004989 +004990 +004991 +004992 +004993 +004994 +004995 +004996 +004997 +004998 +004999 +005000 +005001 +005002 +005003 +005004 +005005 +005006 +005007 +005008 +005009 +005010 +005011 +005012 +005013 +005014 +005015 +005016 +005017 +005018 +005019 +005020 +005021 +005022 +005023 +005024 +005025 +005026 +005027 +005028 +005029 +005030 +005031 +005032 +005033 +005034 +005035 +005036 +005037 +005038 +005039 +005040 +005041 +005042 +005043 +005044 +005045 +005046 +005047 +005048 +005049 +005050 +005051 +005052 +005053 +005054 +005055 +005056 +005057 +005058 +005059 +005060 +005061 +005062 +005063 +005064 +005065 +005066 +005067 +005068 +005069 +005070 +005071 +005072 +005073 +005074 +005075 +005076 +005077 +005078 +005079 +005080 +005081 +005082 +005083 +005084 +005085 +005086 +005087 +005088 +005089 +005090 +005091 +005092 +005093 +005094 +005095 +005096 +005097 +005098 +005099 +005100 +005101 +005102 +005103 +005104 +005105 +005106 +005107 +005108 +005109 +005110 +005111 +005112 +005113 +005114 +005115 +005116 +005117 +005118 +005119 +005120 +005121 +005122 +005123 +005124 +005125 +005126 +005127 +005128 +005129 +005130 +005131 +005132 +005133 +005134 +005135 +005136 +005137 +005138 +005139 +005140 +005141 +005142 +005143 +005144 +005145 +005146 +005147 +005148 +005149 +005150 +005151 +005152 +005153 +005154 +005155 +005156 +005157 +005158 +005159 +005160 +005161 +005162 +005163 +005164 +005165 +005166 +005167 +005168 +005169 +005170 +005171 +005172 +005173 +005174 +005175 +005176 +005177 +005178 +005179 +005180 +005181 +005182 +005183 +005184 +005185 +005186 +005187 +005188 +005189 +005190 +005191 +005192 +005193 +005194 +005195 +005196 +005197 +005198 +005199 +005200 +005201 +005202 +005203 +005204 +005205 +005206 +005207 +005208 +005209 +005210 +005211 +005212 +005213 +005214 +005215 +005216 +005217 +005218 +005219 +005220 +005221 +005222 +005223 +005224 +005225 +005226 +005227 +005228 +005229 +005230 +005231 +005232 +005233 +005234 +005235 +005236 +005237 +005238 +005239 +005240 +005241 +005242 +005243 +005244 +005245 +005246 +005247 +005248 +005249 +005250 +005251 +005252 +005253 +005254 +005255 +005256 +005257 +005258 +005259 +005260 +005261 +005262 +005263 +005264 +005265 +005266 +005267 +005268 +005269 +005270 +005271 +005272 +005273 +005274 +005275 +005276 +005277 +005278 +005279 +005280 +005281 +005282 +005283 +005284 +005285 +005286 +005287 +005288 +005289 +005290 +005291 +005292 +005293 +005294 +005295 +005296 +005297 +005298 +005299 +005300 +005301 +005302 +005303 +005304 +005305 +005306 +005307 +005308 +005309 +005310 +005311 +005312 +005313 +005314 +005315 +005316 +005317 +005318 +005319 +005320 +005321 +005322 +005323 +005324 +005325 +005326 +005327 +005328 +005329 +005330 +005331 +005332 +005333 +005334 +005335 +005336 +005337 +005338 +005339 +005340 +005341 +005342 +005343 +005344 +005345 +005346 +005347 +005348 +005349 +005350 +005351 +005352 +005353 +005354 +005355 +005356 +005357 +005358 +005359 +005360 +005361 +005362 +005363 +005364 +005365 +005366 +005367 +005368 +005369 +005370 +005371 +005372 +005373 +005374 +005375 +005376 +005377 +005378 +005379 +005380 +005381 +005382 +005383 +005384 +005385 +005386 +005387 +005388 +005389 +005390 +005391 +005392 +005393 +005394 +005395 +005396 +005397 +005398 +005399 +005400 +005401 +005402 +005403 +005404 +005405 +005406 +005407 +005408 +005409 +005410 +005411 +005412 +005413 +005414 +005415 +005416 +005417 +005418 +005419 +005420 +005421 +005422 +005423 +005424 +005425 +005426 +005427 +005428 +005429 +005430 +005431 +005432 +005433 +005434 +005435 +005436 +005437 +005438 +005439 +005440 +005441 +005442 +005443 +005444 +005445 +005446 +005447 +005448 +005449 +005450 +005451 +005452 +005453 +005454 +005455 +005456 +005457 +005458 +005459 +005460 +005461 +005462 +005463 +005464 +005465 +005466 +005467 +005468 +005469 +005470 +005471 +005472 +005473 +005474 +005475 +005476 +005477 +005478 +005479 +005480 +005481 +005482 +005483 +005484 +005485 +005486 +005487 +005488 +005489 +005490 +005491 +005492 +005493 +005494 +005495 +005496 +005497 +005498 +005499 +005500 +005501 +005502 +005503 +005504 +005505 +005506 +005507 +005508 +005509 +005510 +005511 +005512 +005513 +005514 +005515 +005516 +005517 +005518 +005519 +005520 +005521 +005522 +005523 +005524 +005525 +005526 +005527 +005528 +005529 +005530 +005531 +005532 +005533 +005534 +005535 +005536 +005537 +005538 +005539 +005540 +005541 +005542 +005543 +005544 +005545 +005546 +005547 +005548 +005549 +005550 +005551 +005552 +005553 +005554 +005555 +005556 +005557 +005558 +005559 +005560 +005561 +005562 +005563 +005564 +005565 +005566 +005567 +005568 +005569 +005570 +005571 +005572 +005573 +005574 +005575 +005576 +005577 +005578 +005579 +005580 +005581 +005582 +005583 +005584 +005585 +005586 +005587 +005588 +005589 +005590 +005591 +005592 +005593 +005594 +005595 +005596 +005597 +005598 +005599 +005600 +005601 +005602 +005603 +005604 +005605 +005606 +005607 +005608 +005609 +005610 +005611 +005612 +005613 +005614 +005615 +005616 +005617 +005618 +005619 +005620 +005621 +005622 +005623 +005624 +005625 +005626 +005627 +005628 +005629 +005630 +005631 +005632 +005633 +005634 +005635 +005636 +005637 +005638 +005639 +005640 +005641 +005642 +005643 +005644 +005645 +005646 +005647 +005648 +005649 +005650 +005651 +005652 +005653 +005654 +005655 +005656 +005657 +005658 +005659 +005660 +005661 +005662 +005663 +005664 +005665 +005666 +005667 +005668 +005669 +005670 +005671 +005672 +005673 +005674 +005675 +005676 +005677 +005678 +005679 +005680 +005681 +005682 +005683 +005684 +005685 +005686 +005687 +005688 +005689 +005690 +005691 +005692 +005693 +005694 +005695 +005696 +005697 +005698 +005699 +005700 +005701 +005702 +005703 +005704 +005705 +005706 +005707 +005708 +005709 +005710 +005711 +005712 +005713 +005714 +005715 +005716 +005717 +005718 +005719 +005720 +005721 +005722 +005723 +005724 +005725 +005726 +005727 +005728 +005729 +005730 +005731 +005732 +005733 +005734 +005735 +005736 +005737 +005738 +005739 +005740 +005741 +005742 +005743 +005744 +005745 +005746 +005747 +005748 +005749 +005750 +005751 +005752 +005753 +005754 +005755 +005756 +005757 +005758 +005759 +005760 +005761 +005762 +005763 +005764 +005765 +005766 +005767 +005768 +005769 +005770 +005771 +005772 +005773 +005774 +005775 +005776 +005777 +005778 +005779 +005780 +005781 +005782 +005783 +005784 +005785 +005786 +005787 +005788 +005789 +005790 +005791 +005792 +005793 +005794 +005795 +005796 +005797 +005798 +005799 +005800 +005801 +005802 +005803 +005804 +005805 +005806 +005807 +005808 +005809 +005810 +005811 +005812 +005813 +005814 +005815 +005816 +005817 +005818 +005819 +005820 +005821 +005822 +005823 +005824 +005825 +005826 +005827 +005828 +005829 +005830 +005831 +005832 +005833 +005834 +005835 +005836 +005837 +005838 +005839 +005840 +005841 +005842 +005843 +005844 +005845 +005846 +005847 +005848 +005849 +005850 +005851 +005852 +005853 +005854 +005855 +005856 +005857 +005858 +005859 +005860 +005861 +005862 +005863 +005864 +005865 +005866 +005867 +005868 +005869 +005870 +005871 +005872 +005873 +005874 +005875 +005876 +005877 +005878 +005879 +005880 +005881 +005882 +005883 +005884 +005885 +005886 +005887 +005888 +005889 +005890 +005891 +005892 +005893 +005894 +005895 +005896 +005897 +005898 +005899 +005900 +005901 +005902 +005903 +005904 +005905 +005906 +005907 +005908 +005909 +005910 +005911 +005912 +005913 +005914 +005915 +005916 +005917 +005918 +005919 +005920 +005921 +005922 +005923 +005924 +005925 +005926 +005927 +005928 +005929 +005930 +005931 +005932 +005933 +005934 +005935 +005936 +005937 +005938 +005939 +005940 +005941 +005942 +005943 +005944 +005945 +005946 +005947 +005948 +005949 +005950 +005951 +005952 +005953 +005954 +005955 +005956 +005957 +005958 +005959 +005960 +005961 +005962 +005963 +005964 +005965 +005966 +005967 +005968 +005969 +005970 +005971 +005972 +005973 +005974 +005975 +005976 +005977 +005978 +005979 +005980 +005981 +005982 +005983 +005984 +005985 +005986 +005987 +005988 +005989 +005990 +005991 +005992 +005993 +005994 +005995 +005996 +005997 +005998 +005999 +006000 +006001 +006002 +006003 +006004 +006005 +006006 +006007 +006008 +006009 +006010 +006011 +006012 +006013 +006014 +006015 +006016 +006017 +006018 +006019 +006020 +006021 +006022 +006023 +006024 +006025 +006026 +006027 +006028 +006029 +006030 +006031 +006032 +006033 +006034 +006035 +006036 +006037 +006038 +006039 +006040 +006041 +006042 +006043 +006044 +006045 +006046 +006047 +006048 +006049 +006050 +006051 +006052 +006053 +006054 +006055 +006056 +006057 +006058 +006059 +006060 +006061 +006062 +006063 +006064 +006065 +006066 +006067 +006068 +006069 +006070 +006071 +006072 +006073 +006074 +006075 +006076 +006077 +006078 +006079 +006080 +006081 +006082 +006083 +006084 +006085 +006086 +006087 +006088 +006089 +006090 +006091 +006092 +006093 +006094 +006095 +006096 +006097 +006098 +006099 +006100 +006101 +006102 +006103 +006104 +006105 +006106 +006107 +006108 +006109 +006110 +006111 +006112 +006113 +006114 +006115 +006116 +006117 +006118 +006119 +006120 +006121 +006122 +006123 +006124 +006125 +006126 +006127 +006128 +006129 +006130 +006131 +006132 +006133 +006134 +006135 +006136 +006137 +006138 +006139 +006140 +006141 +006142 +006143 +006144 +006145 +006146 +006147 +006148 +006149 +006150 +006151 +006152 +006153 +006154 +006155 +006156 +006157 +006158 +006159 +006160 +006161 +006162 +006163 +006164 +006165 +006166 +006167 +006168 +006169 +006170 +006171 +006172 +006173 +006174 +006175 +006176 +006177 +006178 +006179 +006180 +006181 +006182 +006183 +006184 +006185 +006186 +006187 +006188 +006189 +006190 +006191 +006192 +006193 +006194 +006195 +006196 +006197 +006198 +006199 +006200 +006201 +006202 +006203 +006204 +006205 +006206 +006207 +006208 +006209 +006210 +006211 +006212 +006213 +006214 +006215 +006216 +006217 +006218 +006219 +006220 +006221 +006222 +006223 +006224 +006225 +006226 +006227 +006228 +006229 +006230 +006231 +006232 +006233 +006234 +006235 +006236 +006237 +006238 +006239 +006240 +006241 +006242 +006243 +006244 +006245 +006246 +006247 +006248 +006249 +006250 +006251 +006252 +006253 +006254 +006255 +006256 +006257 +006258 +006259 +006260 +006261 +006262 +006263 +006264 +006265 +006266 +006267 +006268 +006269 +006270 +006271 +006272 +006273 +006274 +006275 +006276 +006277 +006278 +006279 +006280 +006281 +006282 +006283 +006284 +006285 +006286 +006287 +006288 +006289 +006290 +006291 +006292 +006293 +006294 +006295 +006296 +006297 +006298 +006299 +006300 +006301 +006302 +006303 +006304 +006305 +006306 +006307 +006308 +006309 +006310 +006311 +006312 +006313 +006314 +006315 +006316 +006317 +006318 +006319 +006320 +006321 +006322 +006323 +006324 +006325 +006326 +006327 +006328 +006329 +006330 +006331 +006332 +006333 +006334 +006335 +006336 +006337 +006338 +006339 +006340 +006341 +006342 +006343 +006344 +006345 +006346 +006347 +006348 +006349 +006350 +006351 +006352 +006353 +006354 +006355 +006356 +006357 +006358 +006359 +006360 +006361 +006362 +006363 +006364 +006365 +006366 +006367 +006368 +006369 +006370 +006371 +006372 +006373 +006374 +006375 +006376 +006377 +006378 +006379 +006380 +006381 +006382 +006383 +006384 +006385 +006386 +006387 +006388 +006389 +006390 +006391 +006392 +006393 +006394 +006395 +006396 +006397 +006398 +006399 +006400 +006401 +006402 +006403 +006404 +006405 +006406 +006407 +006408 +006409 +006410 +006411 +006412 +006413 +006414 +006415 +006416 +006417 +006418 +006419 +006420 +006421 +006422 +006423 +006424 +006425 +006426 +006427 +006428 +006429 +006430 +006431 +006432 +006433 +006434 +006435 +006436 +006437 +006438 +006439 +006440 +006441 +006442 +006443 +006444 +006445 +006446 +006447 +006448 +006449 +006450 +006451 +006452 +006453 +006454 +006455 +006456 +006457 +006458 +006459 +006460 +006461 +006462 +006463 +006464 +006465 +006466 +006467 +006468 +006469 +006470 +006471 +006472 +006473 +006474 +006475 +006476 +006477 +006478 +006479 +006480 +006481 +006482 +006483 +006484 +006485 +006486 +006487 +006488 +006489 +006490 +006491 +006492 +006493 +006494 +006495 +006496 +006497 +006498 +006499 +006500 +006501 +006502 +006503 +006504 +006505 +006506 +006507 +006508 +006509 +006510 +006511 +006512 +006513 +006514 +006515 +006516 +006517 +006518 +006519 +006520 +006521 +006522 +006523 +006524 +006525 +006526 +006527 +006528 +006529 +006530 +006531 +006532 +006533 +006534 +006535 +006536 +006537 +006538 +006539 +006540 +006541 +006542 +006543 +006544 +006545 +006546 +006547 +006548 +006549 +006550 +006551 +006552 +006553 +006554 +006555 +006556 +006557 +006558 +006559 +006560 +006561 +006562 +006563 +006564 +006565 +006566 +006567 +006568 +006569 +006570 +006571 +006572 +006573 +006574 +006575 +006576 +006577 +006578 +006579 +006580 +006581 +006582 +006583 +006584 +006585 +006586 +006587 +006588 +006589 +006590 +006591 +006592 +006593 +006594 +006595 +006596 +006597 +006598 +006599 +006600 +006601 +006602 +006603 +006604 +006605 +006606 +006607 +006608 +006609 +006610 +006611 +006612 +006613 +006614 +006615 +006616 +006617 +006618 +006619 +006620 +006621 +006622 +006623 +006624 +006625 +006626 +006627 +006628 +006629 +006630 +006631 +006632 +006633 +006634 +006635 +006636 +006637 +006638 +006639 +006640 +006641 +006642 +006643 +006644 +006645 +006646 +006647 +006648 +006649 +006650 +006651 +006652 +006653 +006654 +006655 +006656 +006657 +006658 +006659 +006660 +006661 +006662 +006663 +006664 +006665 +006666 +006667 +006668 +006669 +006670 +006671 +006672 +006673 +006674 +006675 +006676 +006677 +006678 +006679 +006680 +006681 +006682 +006683 +006684 +006685 +006686 +006687 +006688 +006689 +006690 +006691 +006692 +006693 +006694 +006695 +006696 +006697 +006698 +006699 +006700 +006701 +006702 +006703 +006704 +006705 +006706 +006707 +006708 +006709 +006710 +006711 +006712 +006713 +006714 +006715 +006716 +006717 +006718 +006719 +006720 +006721 +006722 +006723 +006724 +006725 +006726 +006727 +006728 +006729 +006730 +006731 +006732 +006733 +006734 +006735 +006736 +006737 +006738 +006739 +006740 +006741 +006742 +006743 +006744 +006745 +006746 +006747 +006748 +006749 +006750 +006751 +006752 +006753 +006754 +006755 +006756 +006757 +006758 +006759 +006760 +006761 +006762 +006763 +006764 +006765 +006766 +006767 +006768 +006769 +006770 +006771 +006772 +006773 +006774 +006775 +006776 +006777 +006778 +006779 +006780 +006781 +006782 +006783 +006784 +006785 +006786 +006787 +006788 +006789 +006790 +006791 +006792 +006793 +006794 +006795 +006796 +006797 +006798 +006799 +006800 +006801 +006802 +006803 +006804 +006805 +006806 +006807 +006808 +006809 +006810 +006811 +006812 +006813 +006814 +006815 +006816 +006817 +006818 +006819 +006820 +006821 +006822 +006823 +006824 +006825 +006826 +006827 +006828 +006829 +006830 +006831 +006832 +006833 +006834 +006835 +006836 +006837 +006838 +006839 +006840 +006841 +006842 +006843 +006844 +006845 +006846 +006847 +006848 +006849 +006850 +006851 +006852 +006853 +006854 +006855 +006856 +006857 +006858 +006859 +006860 +006861 +006862 +006863 +006864 +006865 +006866 +006867 +006868 +006869 +006870 +006871 +006872 +006873 +006874 +006875 +006876 +006877 +006878 +006879 +006880 +006881 +006882 +006883 +006884 +006885 +006886 +006887 +006888 +006889 +006890 +006891 +006892 +006893 +006894 +006895 +006896 +006897 +006898 +006899 +006900 +006901 +006902 +006903 +006904 +006905 +006906 +006907 +006908 +006909 +006910 +006911 +006912 +006913 +006914 +006915 +006916 +006917 +006918 +006919 +006920 +006921 +006922 +006923 +006924 +006925 +006926 +006927 +006928 +006929 +006930 +006931 +006932 +006933 +006934 +006935 +006936 +006937 +006938 +006939 +006940 +006941 +006942 +006943 +006944 +006945 +006946 +006947 +006948 +006949 +006950 +006951 +006952 +006953 +006954 +006955 +006956 +006957 +006958 +006959 +006960 +006961 +006962 +006963 +006964 +006965 +006966 +006967 +006968 +006969 +006970 +006971 +006972 +006973 +006974 +006975 +006976 +006977 +006978 +006979 +006980 +006981 +006982 +006983 +006984 +006985 +006986 +006987 +006988 +006989 +006990 +006991 +006992 +006993 +006994 +006995 +006996 +006997 +006998 +006999 +007000 +007001 +007002 +007003 +007004 +007005 +007006 +007007 +007008 +007009 +007010 +007011 +007012 +007013 +007014 +007015 +007016 +007017 +007018 +007019 +007020 +007021 +007022 +007023 +007024 +007025 +007026 +007027 +007028 +007029 +007030 +007031 +007032 +007033 +007034 +007035 +007036 +007037 +007038 +007039 +007040 +007041 +007042 +007043 +007044 +007045 +007046 +007047 +007048 +007049 +007050 +007051 +007052 +007053 +007054 +007055 +007056 +007057 +007058 +007059 +007060 +007061 +007062 +007063 +007064 +007065 +007066 +007067 +007068 +007069 +007070 +007071 +007072 +007073 +007074 +007075 +007076 +007077 +007078 +007079 +007080 +007081 +007082 +007083 +007084 +007085 +007086 +007087 +007088 +007089 +007090 +007091 +007092 +007093 +007094 +007095 +007096 +007097 +007098 +007099 +007100 +007101 +007102 +007103 +007104 +007105 +007106 +007107 +007108 +007109 +007110 +007111 +007112 +007113 +007114 +007115 +007116 +007117 +007118 +007119 +007120 +007121 +007122 +007123 +007124 +007125 +007126 +007127 +007128 +007129 +007130 +007131 +007132 +007133 +007134 +007135 +007136 +007137 +007138 +007139 +007140 +007141 +007142 +007143 +007144 +007145 +007146 +007147 +007148 +007149 +007150 +007151 +007152 +007153 +007154 +007155 +007156 +007157 +007158 +007159 +007160 +007161 +007162 +007163 +007164 +007165 +007166 +007167 +007168 +007169 +007170 +007171 +007172 +007173 +007174 +007175 +007176 +007177 +007178 +007179 +007180 +007181 +007182 +007183 +007184 +007185 +007186 +007187 +007188 +007189 +007190 +007191 +007192 +007193 +007194 +007195 +007196 +007197 +007198 +007199 +007200 +007201 +007202 +007203 +007204 +007205 +007206 +007207 +007208 +007209 +007210 +007211 +007212 +007213 +007214 +007215 +007216 +007217 +007218 +007219 +007220 +007221 +007222 +007223 +007224 +007225 +007226 +007227 +007228 +007229 +007230 +007231 +007232 +007233 +007234 +007235 +007236 +007237 +007238 +007239 +007240 +007241 +007242 +007243 +007244 +007245 +007246 +007247 +007248 +007249 +007250 +007251 +007252 +007253 +007254 +007255 +007256 +007257 +007258 +007259 +007260 +007261 +007262 +007263 +007264 +007265 +007266 +007267 +007268 +007269 +007270 +007271 +007272 +007273 +007274 +007275 +007276 +007277 +007278 +007279 +007280 +007281 +007282 +007283 +007284 +007285 +007286 +007287 +007288 +007289 +007290 +007291 +007292 +007293 +007294 +007295 +007296 +007297 +007298 +007299 +007300 +007301 +007302 +007303 +007304 +007305 +007306 +007307 +007308 +007309 +007310 +007311 +007312 +007313 +007314 +007315 +007316 +007317 +007318 +007319 +007320 +007321 +007322 +007323 +007324 +007325 +007326 +007327 +007328 +007329 +007330 +007331 +007332 +007333 +007334 +007335 +007336 +007337 +007338 +007339 +007340 +007341 +007342 +007343 +007344 +007345 +007346 +007347 +007348 +007349 +007350 +007351 +007352 +007353 +007354 +007355 +007356 +007357 +007358 +007359 +007360 +007361 +007362 +007363 +007364 +007365 +007366 +007367 +007368 +007369 +007370 +007371 +007372 +007373 +007374 +007375 +007376 +007377 +007378 +007379 +007380 +007381 +007382 +007383 +007384 +007385 +007386 +007387 +007388 +007389 +007390 +007391 +007392 +007393 +007394 +007395 +007396 +007397 +007398 +007399 +007400 +007401 +007402 +007403 +007404 +007405 +007406 +007407 +007408 +007409 +007410 +007411 +007412 +007413 +007414 +007415 +007416 +007417 +007418 +007419 +007420 +007421 +007422 +007423 +007424 +007425 +007426 +007427 +007428 +007429 +007430 +007431 +007432 +007433 +007434 +007435 +007436 +007437 +007438 +007439 +007440 +007441 +007442 +007443 +007444 +007445 +007446 +007447 +007448 +007449 +007450 +007451 +007452 +007453 +007454 +007455 +007456 +007457 +007458 +007459 +007460 +007461 +007462 +007463 +007464 +007465 +007466 +007467 +007468 +007469 +007470 +007471 +007472 +007473 +007474 +007475 +007476 +007477 +007478 +007479 \ No newline at end of file diff --git a/second/data/ImageSets/train_debug.txt b/second/data/ImageSets/train_debug.txt new file mode 100644 index 00000000..dec8534e --- /dev/null +++ b/second/data/ImageSets/train_debug.txt @@ -0,0 +1,8 @@ +000000 +000003 +000007 +000009 +000010 +000011 +000012 +000013 \ No newline at end of file diff --git a/second/data/ImageSets/val_debug.txt b/second/data/ImageSets/val_debug.txt new file mode 100644 index 00000000..6c7408f9 --- /dev/null +++ b/second/data/ImageSets/val_debug.txt @@ -0,0 +1,8 @@ +000001 +000002 +000004 +000005 +000006 +000008 +000015 +000019 \ No newline at end of file diff --git a/second/data/csi_dataset.py b/second/data/csi_dataset.py new file mode 100644 index 00000000..96dc6f05 --- /dev/null +++ b/second/data/csi_dataset.py @@ -0,0 +1,153 @@ +"""Run inference on CSI dataset""" +from pathlib import Path +import re + +import cv2 +from google.protobuf import text_format +import lupa +import numpy as np +import torch + +from second.data.kitti_dataset import _homogeneous_coords +from second.data import image_utils +from second.protos import pipeline_pb2 +from second.pytorch.train import build_network +from second.utils import config_tool + +LUA_RUNTIME = lupa.LuaRuntime() +LUA_REGEX = re.compile("^return (.*)$", re.DOTALL) + + +def parse_lua(text): + return LUA_RUNTIME.eval(re.fullmatch(LUA_REGEX, text)[1]) + + +def read_rig_calibration(cal_lua: Path): + cal = parse_lua(cal_lua.read_text()) + baseline = cal.baseline or cal.baseline_m + cam_intrincis = cal.c or cal.cam_intrinsics + fx = cam_intrincis.fx + fy = cam_intrincis.fy + cx = cam_intrincis.cx + cy = cam_intrincis.cy + return {"fx": fx, "fy": fy, "cx": cx, "cy": cy, "baseline": baseline} + + +# tweaked version of kitti_dataset._create_reduced_point_cloud +def point_cloud_from_image_and_disparity(rgb, disp, fx, cx, cy, baseline): + """Compute point cloud from image and disparity""" + h, w = disp.shape + xs, ys = _homogeneous_coords( + w, h, focal_length=fx, optical_center_x=cx, optical_center_y=cy, + ) + xs = np.squeeze(xs) + ys = np.squeeze(ys) + valid_idx = disp > 0 + zs = fx * baseline / disp[valid_idx] + xs = xs[valid_idx] * zs + ys = ys[valid_idx] * zs + + # individual color channels + r = rgb[:, :, 0] + g = rgb[:, :, 1] + b = rgb[:, :, 2] + # make reflections visible in viewer + colors = np.hstack( + [ + b[valid_idx].reshape((-1, 1)), + g[valid_idx].reshape((-1, 1)), + r[valid_idx].reshape((-1, 1)), + ] + ) + points = np.hstack([xs.reshape((-1, 1)), ys.reshape((-1, 1)), zs.reshape((-1, 1))]) + points = np.hstack([points, colors]).astype(np.float32) + + # downsample point clouds density + points = points[0::4, ...] + return points + + +def detect_bboxes(csi_path: Path, checkpoint_path: Path, config_path: Path): + # read calibration + cal_file = csi_path / "calibration/rectified_calibration.lua" + cal = read_rig_calibration(cal_file) + + # create csi image stack readers + disp_reader = image_utils.RawReader( + image_utils.find_image_stack( + csi_path / "training", "_train_x_disp.left.scale16" + ) + ) + rgb_reader = image_utils.RawReader( + image_utils.find_image_stack(csi_path / "training", "_train_rgb.left.sqrt") + ) + + # create network + config = pipeline_pb2.TrainEvalPipelineConfig() + with open(config_path, "r") as f: + proto_str = f.read() + text_format.Merge(proto_str, config) + model_cfg = config.model.second + config_tool.change_detection_range(model_cfg, [-50, -50, 50, 50]) + device = torch.device("cuda") + net = build_network(model_cfg).to(device).eval() + net.load_state_dict(torch.load(checkpoint_path)) + target_assigner = net.target_assigner + voxel_generator = net.voxel_generator + + # generate anchors + grid_size = voxel_generator.grid_size + feature_map_size = grid_size[:2] // config_tool.get_downsample_factor(model_cfg) + feature_map_size = [*feature_map_size, 1][::-1] + + anchors = target_assigner.generate_anchors(feature_map_size)["anchors"] + anchors = torch.tensor(anchors, dtype=torch.float32, device=device) + anchors = anchors.view(1, -1, 7) + + # detect bboxes + num_frames = disp_reader.total_frames + for ix in range(num_frames): + print(f"Processing frame {ix} of {num_frames}...") + + # load frame data + rgb = cv2.resize( + np.squeeze(rgb_reader.get_frames(ix, 1).astype(np.float32) * (1.0 / 255.0)), + (disp_reader.w, disp_reader.h), + ) + disp = np.squeeze(disp_reader.get_frames(ix, 1).astype(np.float32)) + + # process point cloud + points = point_cloud_from_image_and_disparity( + rgb, disp, cal["fx"], cal["cx"], cal["cy"], cal["baseline"] + ) + + # generate voxels + voxel_dict = voxel_generator.generate(points, max_voxels=90000) + voxels = voxel_dict["voxels"] + coords = voxel_dict["coordinates"] + num_points = voxel_dict["num_points_per_voxel"] + coords = np.pad(coords, ((0, 0), (1, 0)), mode="constant", constant_values=0) + voxels = torch.tensor(voxels, dtype=torch.float32, device=device) + coords = torch.tensor(coords, dtype=torch.int32, device=device) + num_points = torch.tensor(num_points, dtype=torch.int32, device=device) + + # make detection + example = { + "anchors": anchors, + "voxels": voxels, + "num_points": num_points, + "coordinates": coords, + } + with torch.no_grad(): + prediction = net(example)[0] + + boxes = prediction["box3d_lidar"].detach().cpu().numpy() + print(boxes) + + +if __name__ == "__main__": + detect_bboxes( + Path("/host/data/csi-stacks/csi-200306-414905"), + Path("/host/data/second/pp_disp_rgb_3classes_model/voxelnet-296960.tckpt"), + Path("/host/data/second/pp_disp_rgb_3classes_model/pipeline.config"), + ) diff --git a/second/data/exr_utils.py b/second/data/exr_utils.py new file mode 100644 index 00000000..c1b11637 --- /dev/null +++ b/second/data/exr_utils.py @@ -0,0 +1,88 @@ +"""Utility functions for working with OpenEXR images.""" + +import pathlib +from typing import Sequence + +import Imath +import numpy as np +import OpenEXR + + +DTYPE_MAP = { + np.float16: Imath.PixelType(Imath.PixelType.HALF), + np.float32: Imath.PixelType(Imath.PixelType.FLOAT), + np.uint32: Imath.PixelType(Imath.PixelType.UINT), +} + +MAP_DTYPE = {str(v):k for k, v in DTYPE_MAP.items()} + +def save( + path: pathlib.Path, + img: np.ndarray, + dtype: np.dtype = None, + channel_names: Sequence = None, + compression: Imath.Compression = Imath.Compression.NO_COMPRESSION, +): + """Save image as an OpenEXR file""" + if dtype is None: + dtype = img.dtype.type + # input image is of shape [height, width, nch] + header = OpenEXR.Header(img.shape[1], img.shape[0]) + if len(img.shape) == 2: + channel_names = ["Y"] + img.shape = [img.shape[0], img.shape[1], 1] + elif channel_names is None: + channel_names = [str(k) for k in range(img.shape[2])] + header["channels"] = { + name: Imath.Channel(DTYPE_MAP[dtype]) for name in channel_names + } + header["compression"] = Imath.Compression(compression) + data = { + name: (img[:, :, k].astype(dtype)).tostring() + for k, name in enumerate(channel_names) + } + + out = OpenEXR.OutputFile(str(path), header) + out.writePixels(data) + out.close() + if len(img.shape) == 3 and img.shape[2] == 1: + img.shape = img.shape[:2] + +def load(path: pathlib.Path, dtype: np.dtype = np.float32): + """Loads the specified OpenEXR image and returns it in numpy format. + + Note: Legacy images were saved as RGB with data only in R channel, so this function + does not currently support RGB images. + """ + infile = OpenEXR.InputFile(str(path)) + header = infile.header() + # load width and height + dw = header["dataWindow"] + width = dw.max.x - dw.min.x + 1 + height = dw.max.y - dw.min.y + 1 + # channel precisions + channel_precision = {c: v.type for c, v in header["channels"].items()} + # load channels + channels = header["channels"] + if "R" in channels: + channel_names = ["R"] + num_chs = 1 + else: + channel_names = list(channels.keys()) + num_chs = len(channels) + + # assume all channels are of the same dtype + dtype = MAP_DTYPE[str(channel_precision[channel_names[0]])] + + if num_chs == 1: + img_bytes = infile.channel(channel_names[0], DTYPE_MAP[dtype]) + img = np.frombuffer(img_bytes, dtype=dtype) + img.shape = (height, width) + else: + img = np.zeros((height, width, num_chs), dtype=dtype) + strings = infile.channels(channel_names) + for i, string in enumerate(strings): + img[:, :, i] = np.frombuffer(string, dtype=dtype).reshape(height, width) + + infile.close() + return img diff --git a/second/data/image_utils.py b/second/data/image_utils.py new file mode 100644 index 00000000..63f68777 --- /dev/null +++ b/second/data/image_utils.py @@ -0,0 +1,515 @@ +"""Utilities for reading image stack data +""" +import os +import glob +import pathlib +import warnings +import numpy as np +from matplotlib import pyplot as plt + +IMAGE_UTILS_TYPE_SELECT = { + "float": np.float32, + "ushort": np.uint16, + "short": np.int16, + "uchar": np.uint8, + "char": np.int8, +} + + +def scale16_tform(x): + x = x.astype(float) * (1.0 / 16.0) + return x + + +IMAGE_UTILS_TFORM_SELECT = { + "id": None, # the identity function (the "do nothing" transform) + "left": None, # take no action, left label is not a transform + "right": None, # take no action, right label is not a transform + "sqrt": None, # take no action, sqrt label transform is handled seperately (for now) + "scale16": scale16_tform, + # TODO: make all the changes in the ML pipeline to allow turning this on + # "sqrt": lambda x: np.square(x.astype(float) * (1.0 / 255.0)), +} + + +def prefix_file_find(folder, prefix): + """find files by prefix""" + hits = sorted(glob.glob(folder + "/**/" + prefix + "*", recursive=True)) + unique_parts = [] + for hit in hits: + unique_parts.append( + hit[hit.find(folder) + len(folder) : hit.rfind(prefix)].rstrip("/") + ) + return hits, unique_parts + + +def find_image_stack( + directory: pathlib.Path, image_prefix: str, recursive: bool = False +) -> str: + """find image stack""" + if recursive: + match = glob.glob(str(directory / "**" / image_prefix) + "*") + else: + match = glob.glob(str(directory / image_prefix) + "*") + match = [x for x in match if not x.endswith(".gz")] # ignore zipped files + if len(match) == 0: + raise Exception( + f"Could not find image with prefix {image_prefix} in {directory}" + ) + elif len(match) > 1: + raise Exception( + f"Found multiple images with prefix {image_prefix} in {directory}, " + "please remove old images" + ) + return match[0] + + +class RawReader: + """Reads images from an image stack file + + Args: + fname: name of image stack file with a format like: + .[].... + E.g: some_image.512.256.3.uchar, or img.sqrt.512.256.3.uchar + + `tform` can be any from `IMAGE_UTILS_TFORM_SELECT` (or missing) + + `type` can be any from `IMAGE_UTILS_TYPE_SELECT` + + TODO: use null instead of "" + + mask_value: if provided, get_frames will return an array masked with + x == mask_value + + first_frame: a frame offset to add to subsequent `get_` calls + + Note: if the image stack filename contains six or more tokens separated by + `.`, and the token in the `` slot does not match a valid `tform` name in + `IMAGE_UTILS_TFORM_SELECT`, a warning will be raised. You may consider adding `id` + to the image stack name to get the same behavior but suppress the warning. + """ + + def __init__(self, fname="", mask_value=None, first_frame=0): + # determine offsets for underlying raw data + self.fname = str(fname) + s = self.fname.split(".") + self.w = int(s[-4]) + self.h = int(s[-3]) + self.nchan = int(s[-2]) + self.type = IMAGE_UTILS_TYPE_SELECT[s[-1]] + self.bytes_per_pix = np.dtype(self.type).itemsize + self.pix_per_frame = self.w * self.h * self.nchan + total_bytes = os.path.getsize(self.fname) + self.total_frames = ( + total_bytes // (self.bytes_per_pix * self.pix_per_frame) - first_frame + ) + self.mask_value = mask_value + self.first_frame = first_frame + + # get the post-processing transform + self.tform = None + if len(s) > 5: + tform_id = s[-5] + if tform_id in IMAGE_UTILS_TFORM_SELECT.keys(): + self.tform = IMAGE_UTILS_TFORM_SELECT[tform_id] + else: + msg = f"Unrecognized transform ID: {tform_id}. Ignoring it." + warnings.warn(msg) + + def get_frames(self, start, nframes): + """read raw data from file""" + nframes = min(nframes, self.total_frames - start) + with open(self.fname) as f: + f.seek(self.pix_per_frame * self.bytes_per_pix * (start + self.first_frame)) + frames = np.fromfile(f, dtype=self.type, count=self.pix_per_frame * nframes) + # don't use reshape; it should be an error to copy data here + frames.shape = (nframes, self.h, self.w, self.nchan) + + if self.mask_value is not None: + # make mask (needs to be done before possible transformation) + mask = frames == self.mask_value + + if self.tform is not None: + # apply transformation + frames = self.tform(frames) + + if self.mask_value is not None: + # apply mask + frames = np.ma.masked_where(mask, frames) + + return frames + + +class TensorReader(RawReader): + """Reads tensors from a tensor stack file + + Individual tensor channels can be accessed through the RawReader interfaces. + + Args: + fname: name of tensor stack file with a format like: + ....1. + E.g: _some_tensor.32.256.128.1.float + `type` can be any from `IMAGE_UTILS_TYPE_SELECT` + + Note: the `.1.` for "colors" allows each tensor channel to be accessed + as frames of the RawReader superclass + + first_frame: a frame offset to add to subsequent `get_` calls + + Attributes: + tensor_channels (int): the number of channels per tensor. Each channel + is an individual frame + total_tensors (int): the total number of tensors available in the file + """ + + def __init__(self, fname="", first_frame=0): + super().__init__(fname, first_frame=first_frame) + self.tensor_channels = int(self.fname.split(".")[-5]) + self.total_tensors = self.total_frames // self.tensor_channels + + def get_shape(self, ntensors=None): + """Returns the shape of the tensor returned by get_tensors. + + (tensors, channels, height, width) + + Args: + ntensors: number of tensors (defaults to total number in the file) + """ + + shape = ( + ntensors or self.total_tensors, + self.tensor_channels, + self.h, + self.w, + ) + return shape + + def get_tensors(self, start=0, ntensors=None): + """Reads tensors from the file + + Args: + start: tensor offset from first_frame (see __init__) + ntensors: number of tensors to retrieve + + Returns: + an array of data read from the file with shape + `self.get_shape(ntensors)` + """ + ntensors = ntensors or self.total_tensors + tensors = super().get_frames( + start * self.tensor_channels, ntensors * self.tensor_channels + ) + tensors.shape = self.get_shape(ntensors) + return tensors + + +def write_raw_data(file_without_ext, data_array, data_type=None, append=False): + """ + write the data array into binary format as those from reconstructed batch + Args: + file_without_ext: output file without final extension + data_array: np array of shape (n, h, w, c) or (n, h, w) to dump + data_type: one of IMAGE_UTILS_TYPE_SELECT + Returns: + None + """ + # reverse mapping from np types to key str + if data_type is None: + data_type2keys = { + IMAGE_UTILS_TYPE_SELECT[key]: key for key in IMAGE_UTILS_TYPE_SELECT + } + array_item_type = type(data_array.flat[0]) + if array_item_type not in data_type2keys: + raise NotImplementedError( + f"array type {array_item_type} is not in the list {str(data_type2keys)}" + ) + data_type = data_type2keys[array_item_type] + # assume it is with at least three dimensions and at most four dimensions + assert ( + len(data_array.shape) <= 4 and len(data_array.shape) >= 3 + ), "input data array needs to be (n,h,w,c) or (n,h,w)" + w = data_array.shape[2] + h = data_array.shape[1] + if len(data_array.shape) <= 3: + nchan = 1 + else: + nchan = data_array.shape[3] + if data_type not in IMAGE_UTILS_TYPE_SELECT: + raise NotImplementedError( + f"data_type {data_type} is not in the list {str(IMAGE_UTILS_TYPE_SELECT)}" + ) + + output_file_name = f"{file_without_ext}.{w}.{h}.{nchan}.{data_type}" + with open(output_file_name, "ab" if append else "wb") as f: + f.write(data_array.copy(order="C")) + + +# unpack raw12 into raw16 +def raw12to16(x): + """expand 12-bit packed samples to 16 bits""" + y = np.zeros((len(x) * 2 // 3,), dtype=np.uint16) + + a_high = x[0::3] + a_low = x[1::3] + a_low = np.left_shift(a_low, 4) + y[0::2] = a_high * 256 + a_low + + b_high = x[2::3] + b_low = x[1::3] + b_low = np.left_shift(np.right_shift(b_low, 4), 4) + y[1::2] = b_high * 256 + b_low + return y + + +class Raw12Reader: + """class to read images from a raw stack of bayer12 + frame output dimension is [nframes, h/2, w/2, 4(rgbg)] + """ + + def __init__(self, fname="", w=1280, h=512, pad=0, rggb=None): + self.fname = fname + self.w = w + self.h = h + self.pad = pad + if rggb is None: + self.rggb = [0, 1, 2, 3] + else: + self.rggb = rggb + self.bytes_per_frame = (self.w * self.h * 3) // 2 + total_bytes = os.path.getsize(fname) + self.total_frames = total_bytes // (self.bytes_per_frame + self.pad) + + def get_frames(self, start, nframes): + """get frames""" + with open(self.fname) as f: + f.seek((self.bytes_per_frame + self.pad) * start) + x = np.fromfile( + f, dtype=np.uint8, count=(self.bytes_per_frame + self.pad) * nframes + ) + + if self.pad > 0: + x = np.reshape(x, (nframes, (self.bytes_per_frame + self.pad))) + x = x[:, self.pad : :] + x = np.reshape(x, (np.size(x),)) + + x = raw12to16(x) + x = np.reshape(x, (nframes, self.h, self.w)) + im = np.zeros((nframes, 4, self.h // 2, self.w // 2), dtype=np.float32) + + r_ind = np.unravel_index(self.rggb[0], (2, 2)) + im[:, 0, :, :] = x[:, r_ind[0] :: 2, r_ind[1] :: 2] + + g_ind = np.unravel_index(self.rggb[1], (2, 2)) + im[:, 1, :, :] = x[:, g_ind[0] :: 2, g_ind[1] :: 2] + + b_ind = np.unravel_index(self.rggb[3], (2, 2)) + im[:, 2, :, :] = x[:, b_ind[0] :: 2, b_ind[1] :: 2] + + g_ind = np.unravel_index(self.rggb[2], (2, 2)) + im[:, 3, :, :] = x[:, g_ind[0] :: 2, g_ind[1] :: 2] + + return np.transpose(im, (0, 2, 3, 1)) * (1 / 2 ** 16) + + +class ImageStackPlot(object): + """display for image stack + + x must be size [nimages, h, w, ncolors] + or x can be a Reader class with method "get_frames(start, nframes)" + by passing a list of ImageStackPlot into friends you can have multiple + synchronized image views + + - mousewheel to step through image stack (+shift key to step faster) + - ctrl + mousewheel to zoom + - click and drag mouse to pan + + TODO: use None instead of "" + """ + + def __init__( + self, x, title="", wintitle="", cmap="", vmin=None, vmax=None, friends=None + ): + # image stack + self.x = x + self.reader = "numpy" not in type(x).__module__ + if self.reader: + self.nframes = self.x.total_frames + title = title or self.x.fname + colors = self.x.nchan + else: + self.nframes = x.shape[0] + colors = x.shape[3] + + wintitle = wintitle or title + + # ui variables + self.ctrl = False # control key is pressed + self.shift = False # shift key is pressed + self.mouse_position = None + self.zoom = 1.0 + self.ind = 0 + + [self.fig, self.ax] = plt.subplots(1, 1, figsize=(15, 6)) + self.ax.set_title(title, fontsize=12) + self.fig.canvas.set_window_title(wintitle) + + first = self.get_frame() + cmap = cmap if cmap != "" else None + self.vmin = vmin + self.vmax = vmax + self.im = plt.imshow(first, vmin=vmin, vmax=vmax, cmap=cmap) + + if colors == 1: + plt.colorbar() + + self.update_frame() + + xlim = self.ax.get_xlim() + self.width = abs(xlim[1] - xlim[0]) + ylim = self.ax.get_ylim() + self.height = abs(ylim[1] - ylim[0]) + + # ui callbacks + self.fig.canvas.mpl_connect("scroll_event", self.scroll_event) + self.fig.canvas.mpl_connect("key_press_event", self.key_press) + self.fig.canvas.mpl_connect("key_release_event", self.key_release) + self.fig.canvas.mpl_connect("button_press_event", self.mouse_press) + self.fig.canvas.mpl_connect("button_release_event", self.mouse_release) + self.fig.canvas.mpl_connect("motion_notify_event", self.mouse_move) + self.fig.canvas.mpl_connect("close_event", self.leave_friends_behind) + self.friends = [] + + # synchronized ImageStackPlots (same zoom and frame #) + if friends is None: + friends = [] + friends.append(self) + friends = list(set(friends)) # ensure each friend is only listed once + for plot in friends: + plot.friends = friends + + def scroll_event(self, event): + """scroll event""" + if self.ctrl and event.xdata is not None: + zoom = self.zoom * (0.5 + 1.5 * (event.button == "up")) + zoom = min(max(zoom, 1), 64) + self.update_zoom( + zoom, event.xdata / self.width, event.ydata / self.height, True + ) + else: + step = -1 + 2 * (event.button == "up") + ind = self.ind + step * (1 + 15 * self.shift) + self.update_frame(ind, True) + + def key_press(self, event): + """key press""" + if event.key == "control": + for plot in self.friends: + plot.ctrl = True + if event.key == "shift": + for plot in self.friends: + plot.shift = True + + def key_release(self, event): + """key release""" + if event.key == "control": + for plot in self.friends: + plot.ctrl = False + if event.key == "shift": + for plot in self.friends: + plot.shift = False + + def mouse_press(self, event): + """mouse press""" + if event.inaxes == self.ax: + self.mouse_position = [event.xdata, event.ydata] + + def mouse_release(self, event): + """mouse release""" + self.mouse_position = None + + def mouse_move(self, event): + """mouse move""" + if self.mouse_position is not None and event.xdata is not None: + dx = (self.mouse_position[0] - event.xdata) / self.width + dy = (self.mouse_position[1] - event.ydata) / self.height + self.update_pan(dx, dy, True) + + def leave_friends_behind(self, event): + """remove the callback references from other figs if user closes the plot""" + self.friends.remove(self) + for plot in self.friends: + plot.friends = self.friends + + def update_pan(self, dx, dy, master=False): + """update pan""" + if master: + for plot in self.friends: + plot.update_pan(dx, dy) + else: + xlim = self.ax.get_xlim() + self.ax.set_xlim([xlim[0] + dx * self.width, xlim[1] + dx * self.width]) + ylim = self.ax.get_ylim() + self.ax.set_ylim([ylim[0] + dy * self.height, ylim[1] + dy * self.height]) + self.im.axes.figure.canvas.draw() + + def update_zoom(self, zoom, x, y, master=False): + """update zoom""" + if master: + for plot in self.friends: + plot.update_zoom(zoom, x, y) + else: + self.zoom = zoom + if self.zoom == 1: + x = 0.5 + y = 0.5 + w2 = 0.5 * self.width / self.zoom + h2 = 0.5 * self.height / self.zoom + self.ax.set_xlim([x * self.width - w2, x * self.width + w2]) + self.ax.set_ylim([y * self.height + h2, y * self.height - h2]) + self.im.axes.figure.canvas.draw() + + def update_frame(self, ind=0, master=False): + """update frame""" + ind = min(max(ind, 0), self.nframes - 1) + if master: + for plot in self.friends: + plot.update_frame(ind) + else: + if ind != self.ind: + self.ind = ind + data = self.get_frame() + self.im.set_data(data) + if self.vmin is None or self.vmax is None: + vmin = self.vmin or np.min(data) + vmax = self.vmax or np.max(data) + self.im.set_clim(vmin=vmin, vmax=vmax) + self.ax.set_xlabel("Frame " + str(self.ind)) + self.im.axes.figure.canvas.draw() + + def get_frame(self): + """get frame""" + if self.reader: + x = self.x.get_frames(start=self.ind, nframes=1) + else: + x = self.x[self.ind, :, :, :] + x = np.expand_dims(x, axis=0) + if x.shape[3] > 3: + x = x[:, :, :, 0:3] + return np.squeeze(x) + + +# example image display +if __name__ == "__main__": + R1 = Raw12Reader("/mnt/nvme/datasets/csi/csi-200102-214954/Left.Bayer12") + R2 = Raw12Reader("/mnt/nvme/datasets/csi/csi-200102-214954/Right.Bayer12") + # R2 = Raw12Reader( + # fname = "/mnt/nvme/datasets/censors/Censors-200102-154806/left.tbayer12", + # w = 2048, h = 800, pad = 32) + + p1 = ImageStackPlot(R1) + p2 = ImageStackPlot(R2, friends=[p1]) + + R = RawReader("/mnt/nvme/datasets/example/rgb.left.640.256.3.float") + p3 = ImageStackPlot(R) + + plt.show() diff --git a/second/data/kitti_common.py b/second/data/kitti_common.py index 5b9e2fe9..ec5cba82 100644 --- a/second/data/kitti_common.py +++ b/second/data/kitti_common.py @@ -85,8 +85,10 @@ def get_kitti_info_path(idx, file_tail='.png', training=True, relative_path=True, - exist_check=True): + exist_check=True, + file_prefix=""): img_idx_str = get_image_index_str(idx) + img_idx_str = file_prefix + img_idx_str img_idx_str += file_tail prefix = pathlib.Path(prefix) if training: @@ -113,6 +115,10 @@ def get_velodyne_path(idx, prefix, training=True, relative_path=True, exist_chec return get_kitti_info_path(idx, prefix, 'velodyne', '.bin', training, relative_path, exist_check) +def get_disparity_path(idx, prefix, training=True, relative_path=True, exist_check=True): + return get_kitti_info_path(idx, prefix, 'disparity','.exr', training, + relative_path, exist_check, file_prefix="disp_x_") + def get_calib_path(idx, prefix, training=True, relative_path=True, exist_check=True): return get_kitti_info_path(idx, prefix, 'calib', '.txt', training, relative_path, exist_check) @@ -162,7 +168,7 @@ def get_kitti_image_info(path, relative_path=True, with_imageshape=True): # image_infos = [] - """ + """ KITTI annotation format version 2: { [optional]points: [N, 3+] point cloud @@ -204,6 +210,10 @@ def map_func(idx): if velodyne: pc_info['velodyne_path'] = get_velodyne_path( idx, path, training, relative_path) + pc_info['disparity_path'] = get_disparity_path( + idx, path, training, relative_path + ) + image_info['image_path'] = get_image_path(idx, path, training, relative_path) if with_imageshape: @@ -250,7 +260,7 @@ def map_func(idx): rect_4x4[:3, :3] = R0_rect else: rect_4x4 = R0_rect - + Tr_velo_to_cam = np.array([ float(info) for info in lines[5].split(' ')[1:13] ]).reshape([3, 4]) @@ -260,6 +270,21 @@ def map_func(idx): if extend_matrix: Tr_velo_to_cam = _extend_matrix(Tr_velo_to_cam) Tr_imu_to_velo = _extend_matrix(Tr_imu_to_velo) + # read camera metrix + camera_p2 = lines[2].split() + assert camera_p2[0] == "P2:" + camera_p2 = [float(x) for x in camera_p2[1:]] + camera_p3 = lines[3].split() + assert camera_p3[0] == "P3:" + camera_p3 = [float(x) for x in camera_p3[1:]] + K = np.array(camera_p2).reshape([3, 4]) + camera_params = { + 'baseline': (camera_p2[3] - camera_p3[3]) / K[0, 0], + 'focal_x': K[0, 0], + 'focal_y': K[1, 1], + 'center_x': K[0, 2], + 'center_y': K[1, 2] + } calib_info['P0'] = P0 calib_info['P1'] = P1 calib_info['P2'] = P2 @@ -267,8 +292,9 @@ def map_func(idx): calib_info['R0_rect'] = rect_4x4 calib_info['Tr_velo_to_cam'] = Tr_velo_to_cam calib_info['Tr_imu_to_velo'] = Tr_imu_to_velo + calib_info['camera_params'] = camera_params info["calib"] = calib_info - + if annotations is not None: info['annos'] = annotations add_difficulty_to_annos(info) diff --git a/second/data/kitti_dataset.py b/second/data/kitti_dataset.py index fe6138c2..3ae3a875 100644 --- a/second/data/kitti_dataset.py +++ b/second/data/kitti_dataset.py @@ -3,31 +3,34 @@ import time from functools import partial +import cv2 import numpy as np from second.core import box_np_ops from second.core import preprocess as prep from second.data import kitti_common as kitti +from second.data import exr_utils as exr_utils from second.utils.eval import get_coco_eval_result, get_official_eval_result from second.data.dataset import Dataset, register_dataset from second.utils.progress_bar import progress_bar_iter as prog_bar @register_dataset class KittiDataset(Dataset): - NumPointFeatures = 4 + NumPointFeatures = 6 def __init__(self, root_path, info_path, class_names=None, prep_func=None, - num_point_features=None): + num_point_features=None, + use_stereo_disp=False): assert info_path is not None with open(info_path, 'rb') as f: infos = pickle.load(f) self._root_path = Path(root_path) self._kitti_infos = infos - + self._use_stereo_disp = use_stereo_disp print("remain number of infos:", len(self._kitti_infos)) self._class_names = class_names self._prep_func = prep_func @@ -111,7 +114,7 @@ def evaluation(self, detections, output_dir): detection When you want to eval your own dataset, you MUST set correct the z axis and box z center. - If you want to eval by my KITTI eval function, you must + If you want to eval by my KITTI eval function, you must provide the correct format annotations. ground_truth_annotations format: { @@ -341,11 +344,11 @@ def _calculate_num_points_in_gt(data_path, annos["num_points_in_gt"] = num_points_in_gt.astype(np.int32) -def create_kitti_info_file(data_path, save_path=None, relative_path=True): +def create_kitti_info_file(data_path, save_path=None, relative_path=True, remove_outside=True): imageset_folder = Path(__file__).resolve().parent / "ImageSets" - train_img_ids = _read_imageset_file(str(imageset_folder / "train.txt")) - val_img_ids = _read_imageset_file(str(imageset_folder / "val.txt")) - test_img_ids = _read_imageset_file(str(imageset_folder / "test.txt")) + train_img_ids = _read_imageset_file(str(imageset_folder / "train.txt")) # "train_debug.txt" + val_img_ids = _read_imageset_file(str(imageset_folder / "val.txt")) # "val_debug.txt" + test_img_ids = _read_imageset_file(str(imageset_folder / "test_debug.txt")) print("Generate info. this may take several minutes.") if save_path is None: @@ -359,7 +362,7 @@ def create_kitti_info_file(data_path, save_path=None, relative_path=True): calib=True, image_ids=train_img_ids, relative_path=relative_path) - _calculate_num_points_in_gt(data_path, kitti_infos_train, relative_path) + _calculate_num_points_in_gt(data_path, kitti_infos_train, relative_path, remove_outside=remove_outside) filename = save_path / 'kitti_infos_train.pkl' print(f"Kitti info train file is saved to {filename}") with open(filename, 'wb') as f: @@ -371,7 +374,7 @@ def create_kitti_info_file(data_path, save_path=None, relative_path=True): calib=True, image_ids=val_img_ids, relative_path=relative_path) - _calculate_num_points_in_gt(data_path, kitti_infos_val, relative_path) + _calculate_num_points_in_gt(data_path, kitti_infos_val, relative_path, remove_outside=remove_outside) filename = save_path / 'kitti_infos_val.pkl' print(f"Kitti info val file is saved to {filename}") with open(filename, 'wb') as f: @@ -395,32 +398,90 @@ def create_kitti_info_file(data_path, save_path=None, relative_path=True): pickle.dump(kitti_infos_test, f) +def _homogeneous_coords( + width, height, focal_length, optical_center_x, optical_center_y +): + """homogeneous coordinates that have same shape as image""" + + positional_coord_x, positional_coord_y = _generate_coords(width=width, height=height) + positional_coord_x = positional_coord_x + positional_coord_y = positional_coord_y + homogeneous_coord_x = (positional_coord_x - optical_center_x) / focal_length + homogeneous_coord_y = (positional_coord_y - optical_center_y) / focal_length + return homogeneous_coord_x, homogeneous_coord_y + + +def _generate_coords(width, height): + """generate coordinates given by width and height""" + + positional_coord_x = np.expand_dims(np.arange(width), [0, 1, 2]) + positional_coord_x = np.tile(positional_coord_x, [height, 1]).astype(np.float32) + + positional_coord_y = np.expand_dims(np.arange(height), [0, 1, 3]) + positional_coord_y = np.tile(positional_coord_y, [1, width]).astype(np.float32) + + return positional_coord_x, positional_coord_y + + def _create_reduced_point_cloud(data_path, info_path, save_path=None, - back=False): + back=False, + use_disparity=False): with open(info_path, 'rb') as f: kitti_infos = pickle.load(f) for info in prog_bar(kitti_infos): pc_info = info["point_cloud"] - image_info = info["image"] - calib = info["calib"] - v_path = pc_info['velodyne_path'] v_path = Path(data_path) / v_path - points_v = np.fromfile( - str(v_path), dtype=np.float32, count=-1).reshape([-1, 4]) + image_info = info["image"] + calib = info["calib"] rect = calib['R0_rect'] P2 = calib['P2'] Trv2c = calib['Tr_velo_to_cam'] - # first remove z < 0 points - # keep = points_v[:, -1] > 0 - # points_v = points_v[keep] - # then remove outside. - if back: - points_v[:, 0] = -points_v[:, 0] - points_v = box_np_ops.remove_outside_points(points_v, rect, Trv2c, P2, - image_info["image_shape"]) + if use_disparity: + camera_params = calib['camera_params'] + disp_path = pc_info['disparity_path'] + disp_path = Path(data_path) / disp_path + # todo: load disparity from exr and convert to point clouds + disp = exr_utils.load(disp_path) + h, w = disp.shape + xs, ys = _homogeneous_coords( + w, + h, + focal_length=camera_params['focal_x'], + optical_center_x=camera_params['center_x'], + optical_center_y=camera_params['center_y'], + ) + xs = np.squeeze(xs) + ys = np.squeeze(ys) + valid_idx = disp > 0 + zs = camera_params['focal_x']*camera_params['baseline'] / disp[valid_idx] + xs = xs[valid_idx]*zs + ys = ys[valid_idx]*zs + # pick g channel + bgr = cv2.imread(str(Path(data_path) / image_info["image_path"])) + b = bgr[:, :, 0] / 255.0 + g = bgr[:, :, 1] / 255.0 + r = bgr[:, :, 2] / 255.0 + # make reflections visible in viewer + colors = np.hstack([b[valid_idx].reshape((-1,1)),g[valid_idx].reshape((-1,1)),r[valid_idx].reshape((-1,1))]) + points_c = np.hstack([ xs.reshape((-1,1)), ys.reshape((-1,1)), zs.reshape((-1,1))]) + points_v = box_np_ops.camera_to_lidar(points_c, rect, Trv2c) + points_v = np.hstack([points_v, colors]).astype(np.float32) + # downsample point clouds density + points_v = points_v[0::4, ...] + else: + points_v = np.fromfile( + str(v_path), dtype=np.float32, count=-1).reshape([-1, 4]) + # first remove z < 0 points + # keep = points_v[:, -1] > 0 + # points_v = points_v[keep] + # then remove outside. + if back: + points_v[:, 0] = -points_v[:, 0] + points_v = box_np_ops.remove_outside_points(points_v, rect, Trv2c, P2, + image_info["image_shape"]) if save_path is None: save_filename = v_path.parent.parent / ( v_path.parent.stem + "_reduced") / v_path.name @@ -440,7 +501,8 @@ def create_reduced_point_cloud(data_path, val_info_path=None, test_info_path=None, save_path=None, - with_back=False): + with_back=False, + use_disparity=False): if train_info_path is None: train_info_path = Path(data_path) / 'kitti_infos_train.pkl' if val_info_path is None: @@ -448,16 +510,16 @@ def create_reduced_point_cloud(data_path, if test_info_path is None: test_info_path = Path(data_path) / 'kitti_infos_test.pkl' - _create_reduced_point_cloud(data_path, train_info_path, save_path) - _create_reduced_point_cloud(data_path, val_info_path, save_path) - _create_reduced_point_cloud(data_path, test_info_path, save_path) + _create_reduced_point_cloud(data_path, train_info_path, save_path, use_disparity=use_disparity) + _create_reduced_point_cloud(data_path, val_info_path, save_path, use_disparity=use_disparity) + _create_reduced_point_cloud(data_path, test_info_path, save_path, use_disparity=use_disparity) if with_back: _create_reduced_point_cloud( - data_path, train_info_path, save_path, back=True) + data_path, train_info_path, save_path, back=True, use_disparity=use_disparity) _create_reduced_point_cloud( - data_path, val_info_path, save_path, back=True) + data_path, val_info_path, save_path, back=True, use_disparity=use_disparity) _create_reduced_point_cloud( - data_path, test_info_path, save_path, back=True) + data_path, test_info_path, save_path, back=True, use_disparity=use_disparity) if __name__ == "__main__": diff --git a/second/kittiviewer/backend/main.py b/second/kittiviewer/backend/main.py index 55b33aae..cc7d233d 100644 --- a/second/kittiviewer/backend/main.py +++ b/second/kittiviewer/backend/main.py @@ -223,7 +223,7 @@ def inference_by_idx(): def main(port=16666): - app.run(host='127.0.0.1', threaded=True, port=port) + app.run(host='10.100.100.230', threaded=True, port=port) if __name__ == '__main__': fire.Fire() diff --git a/second/utils/eval.py b/second/utils/eval.py index 0396ea20..ea4e3a78 100644 --- a/second/utils/eval.py +++ b/second/utils/eval.py @@ -701,7 +701,7 @@ def do_coco_style_eval(gt_annos, min_overlaps = np.zeros([10, *overlap_ranges.shape[1:]]) for i in range(overlap_ranges.shape[1]): for j in range(overlap_ranges.shape[2]): - min_overlaps[:, i, j] = np.linspace(*overlap_ranges[:, i, j]) + min_overlaps[:, i, j] = np.linspace(overlap_ranges[:, i, j][0], overlap_ranges[:, i, j][1], int(overlap_ranges[:, i, j][2])) mAP_bbox, mAP_bev, mAP_3d, mAP_aos = do_eval_v2( gt_annos, dt_annos, @@ -913,4 +913,4 @@ def get_coco_eval_result(gt_annos, return { "result": result, "detail": detail, - } \ No newline at end of file + } diff --git a/second/utils/plot_2d_boxes.py b/second/utils/plot_2d_boxes.py new file mode 100644 index 00000000..050c3e45 --- /dev/null +++ b/second/utils/plot_2d_boxes.py @@ -0,0 +1,144 @@ +"""project 3D kitti boxes to 2D""" +import cv2 +import dataclasses +import fire +from pathlib import Path +import numpy as np +import pickle +import math +from scipy.spatial.transform import Rotation + + +@dataclasses.dataclass +class BBox3D: + label: str + x: float + y: float + z: float + yaw: float + pitch: float + roll: float + height: float + width: float + length: float + det_score: float = -1 + +def apply_camera_matrix(camera_matrix, pts_in): + if camera_matrix.shape[1] == 4: + pts_in = np.concatenate([pts_in, np.ones([pts_in.shape[0], 1])], 1) + pts = np.matmul(camera_matrix, pts_in.T).T + pts = np.stack([pts[:, 0] / pts[:, 2], pts[:, 1] / pts[:, 2]], 1) + return pts + + +def bbox3d_to_cv_points(bbox_3d, camera_matrix): + pts_3d = [] + for i in range(8): + h = -bbox_3d.height if i % 2 == 0 else 0 # bbox_3d.height + w = -bbox_3d.width / 2 if (i // 2) % 2 == 0 else bbox_3d.width / 2 + l = -bbox_3d.length / 2 if (i // 4) % 2 == 0 else bbox_3d.length / 2 + pts_3d.append(np.array([w, h, l])) + + pts_3d = np.stack(pts_3d) + R = Rotation.from_euler("YXZ", [bbox_3d.yaw, bbox_3d.pitch, bbox_3d.roll]) + R = R.as_matrix() + T = np.array([bbox_3d.x, bbox_3d.y, bbox_3d.z]) + pts_3d = np.matmul(R, pts_3d.T).T + T.reshape([1, -1]) + pts = apply_camera_matrix(camera_matrix, pts_3d) + center = apply_camera_matrix(camera_matrix, T.reshape([1, 3])) + return pts, center[0] + +def draw_bbox3d(img, bbox_3d, K, color=None): + if color is None: + color = (255, 0, 0) + corner_pts, center_pt = bbox3d_to_cv_points(bbox_3d, K) + corner_pts = np.round(corner_pts).astype(np.int32) + for i in range(8): + for j in range(8): + if i < j and i ^ j in [1, 2, 4]: + img = cv2.line( + img, corner_pts[i], corner_pts[j], color=color, thickness=1 + ) + + center_pt = np.round(center_pt).astype(np.int32) + img = cv2.circle( + img, (center_pt[0], center_pt[1]), radius=1, color=color, thickness=-1 + ) + return img + + +def read_calib(calib_path): + with open(calib_path, "r") as f: + calib = f.readlines() + P2 = calib[2].split() + assert P2[0] == "P2:" + P2 = [float(x) for x in P2[1:]] + P3 = calib[3].split() + assert P3[0] == "P3:" + P3 = [float(x) for x in P3[1:]] + K = np.array(P2).reshape([3, 4]) + baseline = (P2[3] - P3[3]) / K[0, 0] + return K, baseline + +def plot_box(bbox_3ds, rgb, K, output_path, colors): + for bbox_3d in bbox_3ds: + rgb = draw_bbox3d(rgb, bbox_3d, K, color=colors[bbox_3d.label]) + cv2.imwrite(str(output_path), cv2.cvtColor(rgb, cv2.COLOR_BGR2RGB)) + # print("output to ", output_path) + +def run(): + pkl_path = "/host/ssd/quan/data/tmp/kitti3d_second/disp_rgb_3classes_model/results/step_296960/result.pkl" + # pkl_path = "/host/ssd/quan/data/tmp/kitti3d_second/disp_pp_xyres_20_models_rgb/results/step_296960/result.pkl" + # rgb_path = "/host/ssd/quan/data/tmp/kitti3d_second/training/image_2/000008.png" + # calib_path = "/host/ssd/quan/data/tmp/kitti3d_second/training/calib/000008.txt" + # output_path = "/host/tmp/0000008_boxes.png" + xyz_hwl_r = [ 8.2277, 1.1708, -0.8107, 1.5976, 3.8149, 1.5407, -1.2763] + with open(pkl_path, "rb") as fp: + d = pickle.load(fp) + + rgb_dir = Path("/host/ssd/quan/data/tmp/kitti3d_second/training/image_2") + calib_dir = Path("/host/ssd/quan/data/tmp/kitti3d_second/training/calib/") + output_dir = Path("/host/ssd/quan/data/tmp/kitti_results_cars") + + bbox_3ds = [] + class_names = ["car", "ped", "cyclist"] + colors = {"car": (255,0,0), "ped":(255, 255, 0), "cyclist":(0, 255,255)} + for frame in d: + image_id = frame["metadata"]["image_idx"] + image_name = str(image_id).zfill(6) + rgb_path = rgb_dir / f"{image_name}.png" + calib_path = calib_dir / f"{image_name}.txt" + bbox_3ds = [] + for xyz_hwl_r, score, label in zip(frame["box3d_lidar"], frame["scores"],frame["label_preds"]): + + if label == 0 and score < 0.35: + continue + elif label == 1 and score < 0.35: + continue + elif label == 2 and score < 0.15: + continue + height_start = float(xyz_hwl_r[5])/2 + bbox_3d = BBox3D( + label=class_names[label], + height=float(xyz_hwl_r[5]), + width=float(xyz_hwl_r[3]), + length=float(xyz_hwl_r[4]), + x=-float(xyz_hwl_r[1]), + y=-float(xyz_hwl_r[2])+height_start, + z=float(xyz_hwl_r[0]), + yaw=float(xyz_hwl_r[6]) + math.pi / 2, + pitch=0, + roll=0, + ) + bbox_3ds.append(bbox_3d) + + K, baseline = read_calib(calib_path) + bgr = cv2.imread(str(rgb_path)) + rgb = cv2.cvtColor(bgr, cv2.COLOR_BGR2RGB) + output_path = output_dir / f"{image_name}_boxes.jpg" + plot_box(bbox_3ds, rgb, K, output_path, colors) + if image_id % 100 == 0: + print(f"finished {image_id+1} frames out of {len(d)} in {output_dir}") + +if __name__ == '__main__': + run()