Upload data#

Links: notebook, html, PDF, python, slides, GitHub

Material for the hackathon ENSAE / Red Cross / 2015. Upload data to a blob storage.

from jyquickhelper import add_notebook_menu
add_notebook_menu()

Connexion to the blob storage#

from pyquickhelper.loghelper import get_password
hackathon = {}
hackathon["blob_storage"] = get_password("CRCREDENTIALS", "blobstorage")
hackathon["password"] = get_password("CRCREDENTIALS", "password")

blobstorage = hackathon["blob_storage"]
blobpassword = hackathon["password"]

%load_ext pyensae

cl, bs = %blob_open
cl, bs

(<pyensae.remote.azure_connection.AzureClient at 0x925c2e8>,
 <azure.storage.blob.blobservice.BlobService at 0x925c320>)

%blob_containers

['croix-rouge']

Upload data#

%%file readme.txt

This folder conains data for the Red Cross.

Writing readme.txt

%blob_up readme.txt croix-rouge/readme.txt

'readme.txt'

%blob_ls croix-rouge
name last_modified content_type content_length blob_type
0 build/SINVOICE_M.csv Wed, 18 Nov 2015 18:56:27 GMT application/octet-stream 533771533 BlockBlob
1 data/ITMMASTER.schema.txt Mon, 16 Nov 2015 23:00:34 GMT application/octet-stream 5658 BlockBlob
2 data/ITMMASTER.txt Mon, 09 Nov 2015 21:41:00 GMT application/octet-stream 103096479 BlockBlob
3 data/SINVOICE.schema.txt Mon, 16 Nov 2015 23:00:35 GMT application/octet-stream 10252 BlockBlob
4 data/SINVOICE.txt Mon, 09 Nov 2015 21:42:32 GMT application/octet-stream 1362433753 BlockBlob
5 data/SINVOICEV.schema.txt Mon, 16 Nov 2015 23:00:35 GMT application/octet-stream 7999 BlockBlob
6 data/SINVOICEV.txt Mon, 09 Nov 2015 21:44:08 GMT application/octet-stream 1252461865 BlockBlob
7 data/enseignes_france.csv Mon, 09 Nov 2015 21:40:54 GMT application/octet-stream 6303836 BlockBlob
8 data/stojou.csv Mon, 09 Nov 2015 21:55:23 GMT application/octet-stream 8821375868 BlockBlob
9 readme.txt Sun, 22 Nov 2015 18:00:34 GMT application/octet-stream 45 BlockBlob 
import os
files = ["ITMMASTER.schema.txt", "SINVOICE.schema.txt", "SINVOICEV.schema.txt"]
for file in files:
    print("upload", file, os.stat(file).st_size)
    cl.upload(bs, "croix-rouge", "data/" + file, file)

upload ITMMASTER.schema.txt 5658
upload SINVOICE.schema.txt 10252
upload SINVOICEV.schema.txt 7999

import os
files = ["enseignes_france.csv", "ITMMASTER.txt", "SINVOICE.txt", "SINVOICEV.txt",
         "stojou.csv"]
for file in files:
    print("upload", file, os.stat(file).st_size)
    cl.upload(bs, "croix-rouge", "data/" + file, file)

upload enseignes_france.csv 6303836
upload ITMMASTER.txt 103096479
upload SINVOICE.txt 1362433753
upload SINVOICEV.txt 1252461865
upload stojou.csv 8821375868

import os
files = ["SINVOICE_.txt", "SINVOICEV_.txt"]
for file in files:
    print("upload", file, os.stat(file).st_size)
    cl.upload(bs, "croix-rouge", "data/" + file, file)

upload SINVOICE_.txt 1362433753
upload SINVOICEV_.txt 1252461865

%blob_ls croix-rouge/data
name last_modified content_type content_length blob_type
0 data/ITMMASTER.schema.txt Mon, 16 Nov 2015 23:00:34 GMT application/octet-stream 5658 BlockBlob
1 data/ITMMASTER.txt Mon, 09 Nov 2015 21:41:00 GMT application/octet-stream 103096479 BlockBlob
2 data/SINVOICE.schema.txt Mon, 16 Nov 2015 23:00:35 GMT application/octet-stream 10252 BlockBlob
3 data/SINVOICE.txt Mon, 09 Nov 2015 21:42:32 GMT application/octet-stream 1362433753 BlockBlob
4 data/SINVOICEV.schema.txt Mon, 16 Nov 2015 23:00:35 GMT application/octet-stream 7999 BlockBlob
5 data/SINVOICEV.txt Mon, 09 Nov 2015 21:44:08 GMT application/octet-stream 1252461865 BlockBlob
6 data/enseignes_france.csv Mon, 09 Nov 2015 21:40:54 GMT application/octet-stream 6303836 BlockBlob
7 data/stojou.csv Mon, 09 Nov 2015 21:55:23 GMT application/octet-stream 8821375868 BlockBlob

Upload preprocessed data#

%blob_up SINVOICE_M.csv croix-rouge/build/SINVOICE_M.csv

'build/SINVOICE_M.csv'

%blob_up Test_CRFFOR.GACCTMPD.csv croix-rouge/build/Test_CRFFOR.GACCTMPD.csv

'build/Test_CRFFOR.GACCTMPD.csv'

%blob_ls croix-rouge/build
name last_modified content_type content_length blob_type
0 build/SINVOICE_M.csv Wed, 18 Nov 2015 18:56:27 GMT application/octet-stream 533771533 BlockBlob
1 build/Test_CRFFOR.GACCTMPD.csv Sun, 22 Nov 2015 21:53:38 GMT application/octet-stream 822231942 BlockBlob

Close connection#

%blob_close

True