{"cells": [{"cell_type": "markdown", "metadata": {}, "source": ["# Upload data\n", "\n", "Material for the hackathon ENSAE / Red Cross / 2015. Upload data to a blob storage."]}, {"cell_type": "code", "execution_count": 1, "metadata": {"collapsed": false}, "outputs": [{"data": {"text/html": ["
run previous cell, wait for 2 seconds
\n", ""], "text/plain": [""]}, "execution_count": 2, "metadata": {}, "output_type": "execute_result"}], "source": ["from jyquickhelper import add_notebook_menu\n", "add_notebook_menu()"]}, {"cell_type": "markdown", "metadata": {}, "source": ["## Connexion to the blob storage"]}, {"cell_type": "code", "execution_count": 2, "metadata": {"collapsed": false}, "outputs": [], "source": ["from pyquickhelper.loghelper import get_password\n", "hackathon = {}\n", "hackathon[\"blob_storage\"] = get_password(\"CRCREDENTIALS\", \"blobstorage\")\n", "hackathon[\"password\"] = get_password(\"CRCREDENTIALS\", \"password\")"]}, {"cell_type": "code", "execution_count": 3, "metadata": {"collapsed": false}, "outputs": [], "source": ["blobstorage = hackathon[\"blob_storage\"]\n", "blobpassword = hackathon[\"password\"]"]}, {"cell_type": "code", "execution_count": 4, "metadata": {"collapsed": true}, "outputs": [], "source": ["%load_ext pyensae"]}, {"cell_type": "code", "execution_count": 5, "metadata": {"collapsed": false}, "outputs": [{"data": {"text/plain": ["(,\n", " )"]}, "execution_count": 6, "metadata": {}, "output_type": "execute_result"}], "source": ["cl, bs = %blob_open\n", "cl, bs"]}, {"cell_type": "code", "execution_count": 6, "metadata": {"collapsed": false}, "outputs": [{"data": {"text/plain": ["['croix-rouge']"]}, "execution_count": 7, "metadata": {}, "output_type": "execute_result"}], "source": ["%blob_containers"]}, {"cell_type": "markdown", "metadata": {}, "source": ["## Upload data"]}, {"cell_type": "code", "execution_count": 7, "metadata": {"collapsed": false}, "outputs": [{"name": "stdout", "output_type": "stream", "text": ["Writing readme.txt\n"]}], "source": ["%%file readme.txt\n", "\n", "This folder conains data for the Red Cross."]}, {"cell_type": "code", "execution_count": 8, "metadata": {"collapsed": false}, "outputs": [{"data": {"text/plain": ["'readme.txt'"]}, "execution_count": 9, "metadata": {}, "output_type": "execute_result"}], "source": ["%blob_up readme.txt croix-rouge/readme.txt"]}, {"cell_type": "code", "execution_count": 9, "metadata": {"collapsed": false}, "outputs": [{"data": {"text/html": ["
\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
namelast_modifiedcontent_typecontent_lengthblob_type
0build/SINVOICE_M.csvWed, 18 Nov 2015 18:56:27 GMTapplication/octet-stream533771533BlockBlob
1data/ITMMASTER.schema.txtMon, 16 Nov 2015 23:00:34 GMTapplication/octet-stream5658BlockBlob
2data/ITMMASTER.txtMon, 09 Nov 2015 21:41:00 GMTapplication/octet-stream103096479BlockBlob
3data/SINVOICE.schema.txtMon, 16 Nov 2015 23:00:35 GMTapplication/octet-stream10252BlockBlob
4data/SINVOICE.txtMon, 09 Nov 2015 21:42:32 GMTapplication/octet-stream1362433753BlockBlob
5data/SINVOICEV.schema.txtMon, 16 Nov 2015 23:00:35 GMTapplication/octet-stream7999BlockBlob
6data/SINVOICEV.txtMon, 09 Nov 2015 21:44:08 GMTapplication/octet-stream1252461865BlockBlob
7data/enseignes_france.csvMon, 09 Nov 2015 21:40:54 GMTapplication/octet-stream6303836BlockBlob
8data/stojou.csvMon, 09 Nov 2015 21:55:23 GMTapplication/octet-stream8821375868BlockBlob
9readme.txtSun, 22 Nov 2015 18:00:34 GMTapplication/octet-stream45BlockBlob
\n", "
"], "text/plain": [" name last_modified \\\n", "0 build/SINVOICE_M.csv Wed, 18 Nov 2015 18:56:27 GMT \n", "1 data/ITMMASTER.schema.txt Mon, 16 Nov 2015 23:00:34 GMT \n", "2 data/ITMMASTER.txt Mon, 09 Nov 2015 21:41:00 GMT \n", "3 data/SINVOICE.schema.txt Mon, 16 Nov 2015 23:00:35 GMT \n", "4 data/SINVOICE.txt Mon, 09 Nov 2015 21:42:32 GMT \n", "5 data/SINVOICEV.schema.txt Mon, 16 Nov 2015 23:00:35 GMT \n", "6 data/SINVOICEV.txt Mon, 09 Nov 2015 21:44:08 GMT \n", "7 data/enseignes_france.csv Mon, 09 Nov 2015 21:40:54 GMT \n", "8 data/stojou.csv Mon, 09 Nov 2015 21:55:23 GMT \n", "9 readme.txt Sun, 22 Nov 2015 18:00:34 GMT \n", "\n", " content_type content_length blob_type \n", "0 application/octet-stream 533771533 BlockBlob \n", "1 application/octet-stream 5658 BlockBlob \n", "2 application/octet-stream 103096479 BlockBlob \n", "3 application/octet-stream 10252 BlockBlob \n", "4 application/octet-stream 1362433753 BlockBlob \n", "5 application/octet-stream 7999 BlockBlob \n", "6 application/octet-stream 1252461865 BlockBlob \n", "7 application/octet-stream 6303836 BlockBlob \n", "8 application/octet-stream 8821375868 BlockBlob \n", "9 application/octet-stream 45 BlockBlob "]}, "execution_count": 10, "metadata": {}, "output_type": "execute_result"}], "source": ["%blob_ls croix-rouge"]}, {"cell_type": "code", "execution_count": 10, "metadata": {"collapsed": false}, "outputs": [{"name": "stdout", "output_type": "stream", "text": ["upload ITMMASTER.schema.txt 5658\n", "upload SINVOICE.schema.txt 10252\n", "upload SINVOICEV.schema.txt 7999\n"]}], "source": ["import os\n", "files = [\"ITMMASTER.schema.txt\", \"SINVOICE.schema.txt\", \"SINVOICEV.schema.txt\"]\n", "for file in files:\n", " print(\"upload\", file, os.stat(file).st_size)\n", " cl.upload(bs, \"croix-rouge\", \"data/\" + file, file)"]}, {"cell_type": "code", "execution_count": 11, "metadata": {"collapsed": false}, "outputs": [{"name": "stdout", "output_type": "stream", "text": ["upload enseignes_france.csv 6303836\n", "upload ITMMASTER.txt 103096479\n", "upload SINVOICE.txt 1362433753\n", "upload SINVOICEV.txt 1252461865\n", "upload stojou.csv 8821375868\n"]}], "source": ["import os\n", "files = [\"enseignes_france.csv\", \"ITMMASTER.txt\", \"SINVOICE.txt\", \"SINVOICEV.txt\",\n", " \"stojou.csv\"]\n", "for file in files:\n", " print(\"upload\", file, os.stat(file).st_size)\n", " cl.upload(bs, \"croix-rouge\", \"data/\" + file, file)"]}, {"cell_type": "code", "execution_count": 12, "metadata": {"collapsed": false}, "outputs": [{"name": "stdout", "output_type": "stream", "text": ["upload SINVOICE_.txt 1362433753\n", "upload SINVOICEV_.txt 1252461865\n"]}], "source": ["import os\n", "files = [\"SINVOICE_.txt\", \"SINVOICEV_.txt\"]\n", "for file in files:\n", " print(\"upload\", file, os.stat(file).st_size)\n", " cl.upload(bs, \"croix-rouge\", \"data/\" + file, file)"]}, {"cell_type": "code", "execution_count": 13, "metadata": {"collapsed": false}, "outputs": [{"data": {"text/html": ["
\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
namelast_modifiedcontent_typecontent_lengthblob_type
0data/ITMMASTER.schema.txtMon, 16 Nov 2015 23:00:34 GMTapplication/octet-stream5658BlockBlob
1data/ITMMASTER.txtMon, 09 Nov 2015 21:41:00 GMTapplication/octet-stream103096479BlockBlob
2data/SINVOICE.schema.txtMon, 16 Nov 2015 23:00:35 GMTapplication/octet-stream10252BlockBlob
3data/SINVOICE.txtMon, 09 Nov 2015 21:42:32 GMTapplication/octet-stream1362433753BlockBlob
4data/SINVOICEV.schema.txtMon, 16 Nov 2015 23:00:35 GMTapplication/octet-stream7999BlockBlob
5data/SINVOICEV.txtMon, 09 Nov 2015 21:44:08 GMTapplication/octet-stream1252461865BlockBlob
6data/enseignes_france.csvMon, 09 Nov 2015 21:40:54 GMTapplication/octet-stream6303836BlockBlob
7data/stojou.csvMon, 09 Nov 2015 21:55:23 GMTapplication/octet-stream8821375868BlockBlob
\n", "
"], "text/plain": [" name last_modified \\\n", "0 data/ITMMASTER.schema.txt Mon, 16 Nov 2015 23:00:34 GMT \n", "1 data/ITMMASTER.txt Mon, 09 Nov 2015 21:41:00 GMT \n", "2 data/SINVOICE.schema.txt Mon, 16 Nov 2015 23:00:35 GMT \n", "3 data/SINVOICE.txt Mon, 09 Nov 2015 21:42:32 GMT \n", "4 data/SINVOICEV.schema.txt Mon, 16 Nov 2015 23:00:35 GMT \n", "5 data/SINVOICEV.txt Mon, 09 Nov 2015 21:44:08 GMT \n", "6 data/enseignes_france.csv Mon, 09 Nov 2015 21:40:54 GMT \n", "7 data/stojou.csv Mon, 09 Nov 2015 21:55:23 GMT \n", "\n", " content_type content_length blob_type \n", "0 application/octet-stream 5658 BlockBlob \n", "1 application/octet-stream 103096479 BlockBlob \n", "2 application/octet-stream 10252 BlockBlob \n", "3 application/octet-stream 1362433753 BlockBlob \n", "4 application/octet-stream 7999 BlockBlob \n", "5 application/octet-stream 1252461865 BlockBlob \n", "6 application/octet-stream 6303836 BlockBlob \n", "7 application/octet-stream 8821375868 BlockBlob "]}, "execution_count": 14, "metadata": {}, "output_type": "execute_result"}], "source": ["%blob_ls croix-rouge/data"]}, {"cell_type": "markdown", "metadata": {}, "source": ["## Upload preprocessed data"]}, {"cell_type": "code", "execution_count": 14, "metadata": {"collapsed": false}, "outputs": [{"data": {"text/plain": ["'build/SINVOICE_M.csv'"]}, "execution_count": 15, "metadata": {}, "output_type": "execute_result"}], "source": ["%blob_up SINVOICE_M.csv croix-rouge/build/SINVOICE_M.csv"]}, {"cell_type": "code", "execution_count": 15, "metadata": {"collapsed": false}, "outputs": [{"data": {"text/plain": ["'build/Test_CRFFOR.GACCTMPD.csv'"]}, "execution_count": 16, "metadata": {}, "output_type": "execute_result"}], "source": ["%blob_up Test_CRFFOR.GACCTMPD.csv croix-rouge/build/Test_CRFFOR.GACCTMPD.csv"]}, {"cell_type": "code", "execution_count": 16, "metadata": {"collapsed": false}, "outputs": [{"data": {"text/html": ["
\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
namelast_modifiedcontent_typecontent_lengthblob_type
0build/SINVOICE_M.csvWed, 18 Nov 2015 18:56:27 GMTapplication/octet-stream533771533BlockBlob
1build/Test_CRFFOR.GACCTMPD.csvSun, 22 Nov 2015 21:53:38 GMTapplication/octet-stream822231942BlockBlob
\n", "
"], "text/plain": [" name last_modified \\\n", "0 build/SINVOICE_M.csv Wed, 18 Nov 2015 18:56:27 GMT \n", "1 build/Test_CRFFOR.GACCTMPD.csv Sun, 22 Nov 2015 21:53:38 GMT \n", "\n", " content_type content_length blob_type \n", "0 application/octet-stream 533771533 BlockBlob \n", "1 application/octet-stream 822231942 BlockBlob "]}, "execution_count": 17, "metadata": {}, "output_type": "execute_result"}], "source": ["%blob_ls croix-rouge/build"]}, {"cell_type": "markdown", "metadata": {}, "source": ["## Close connection"]}, {"cell_type": "code", "execution_count": 17, "metadata": {"collapsed": false}, "outputs": [{"data": {"text/plain": ["True"]}, "execution_count": 18, "metadata": {}, "output_type": "execute_result"}], "source": ["%blob_close"]}, {"cell_type": "code", "execution_count": 18, "metadata": {"collapsed": true}, "outputs": [], "source": []}], "metadata": {"kernelspec": {"display_name": "Python 3", "language": "python", "name": "python3"}, "language_info": {"codemirror_mode": {"name": "ipython", "version": 3}, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.5.2"}}, "nbformat": 4, "nbformat_minor": 2}