As the pandas reads timestamp data and converts to HDF5 (HDF5)

0

I am creating a C # application that generates a hdf5 document in which one of the columns in the dataset is a timestamp (using unix), and I would like this file to be read by the pandas so that it reads the timestamp as a datetime64 (which is the format of pandas). I tried to create the file using opandas and realized that in creating the dataset it inserts several attributes that I assumed the pandas read them and process the information and convert from timestamp to datetime, however generating it by C # I can not "cheat" the pandas as if the file had been created by pandas

Here is my method that creates the dataset:

       public void CriaHDF5Customizado(PackingConfigFile pf)
    {

       // PopulaOPC(pf);

        H5FileId arquivoHDF5 = H5F.create("C:/Users/arthuro/Desktop/timestampteste.h5", H5F.CreateMode.ACC_TRUNC);
        H5GroupId datasetGroup = H5G.create(arquivoHDF5, "Datasets");
        H5GroupId infosGroup = H5G.create(arquivoHDF5, "Informations");
        H5G.close(infosGroup);

        opcSt opHelper = new opcSt();
        opHelper.dt = (Int64)DateTime.Parse("1996-11-9").Subtract(new DateTime(1970, 1, 1,12,00,00, DateTimeKind.Utc)).TotalSeconds;
        opHelper.qlt = (Int64)DateTime.Now.Subtract(new DateTime(1970, 1, 1, 0, 0, 0, DateTimeKind.Utc)).TotalSeconds;
        opHelper.vl = 123456.7f;


        int structsize = Marshal.SizeOf(opHelper);
        H5DataTypeId myOPCType = H5T.create(H5T.CreateClass.COMPOUND, structsize);
        //H5T.insert(myOPCType, "TimeStamp", 0, new H5DataTypeId(H5T.H5Type.NATIVE_LONG));            
        H5T.insert(myOPCType, "Quality", 8, new H5DataTypeId(H5T.H5Type.NATIVE_LONG));
        H5T.insert(myOPCType, "Value", 16, new H5DataTypeId(H5T.H5Type.NATIVE_FLOAT));


        long[] ds = new long[1];
        ds[0] = 1;
        H5DataTypeId dtDatetime = H5T.create_array(new H5DataTypeId(H5T.H5Type.NATIVE_LLONG), ds);
        H5T.insert(myOPCType, "TimeStamp", 0, dtDatetime);

        foreach (BasicVariable bv in pf.basicVariableList.bvList)
        {
            bv.PopulateOPCUA();
            var bvnow = bv;
            var bvNro = pf.basicVariableList.bvList.IndexOf(bv);
            long[] dims = new long[1];
            dims[0] = bv.bvData.Count;
            H5DataSpaceId myDataSpace = H5S.create_simple(1, dims);
            H5DataSetId bvDset = H5D.create(datasetGroup, bv.bvTag, myOPCType, myDataSpace);
            var arrayaux = new List<opcSt>();
            foreach (OPC_UA opc in bv.bvData)
            {
                var aux = new opcSt(opc.timeStamp, opc.quality, (float)opc.data);
                arrayaux.Add(aux);
            }
            H5D.write(bvDset, myOPCType, new H5Array<opcSt>(arrayaux.ToArray()));
            string[] stringteste = { "datetime64" };
            H5DataTypeId attrDt = H5T.copy(H5T.H5Type.C_S1);
            H5T.setSize(attrDt, stringteste[0].Length);
            var longsz = new long[] { 1 };
            var enc = new System.Text.ASCIIEncoding();
            var array1 = enc.GetBytes(stringteste[0]);
            var charArray = new byte[stringteste[0].Length + 1];
            array1.CopyTo(charArray, 0);
            charArray[stringteste[0].Length] = 0;
            H5DataSpaceId atribDS = H5S.create_simple(1, longsz);

            H5AttributeId Attribs = H5A.create(bvDset, "TimeStamp_dtype", attrDt, atribDS);
            H5A.write(Attribs, attrDt, new H5Array<byte>(charArray));

            stringteste[0] = "(lp0L0La.";

            H5T.setSize(attrDt, stringteste[0].Length);
            longsz = new long[] { 1 };
            enc = new System.Text.ASCIIEncoding();
            array1 = enc.GetBytes(stringteste[0]);
            charArray = new byte[stringteste[0].Length + 1];
            array1.CopyTo(charArray, 0);
            charArray[stringteste[0].Length] = 0;
            atribDS = H5S.create_simple(1, longsz);

            H5AttributeId Attribs2 = H5A.create(bvDset, "TimeStamp_kind", attrDt, atribDS);
            H5A.write(Attribs2, attrDt, new H5Array<byte>(charArray));

            stringteste[0] = "N.";

            H5T.setSize(attrDt, stringteste[0].Length);
            longsz = new long[] { 1 };
            enc = new System.Text.ASCIIEncoding();
            array1 = enc.GetBytes(stringteste[0]);
            charArray = new byte[stringteste[0].Length + 1];
            array1.CopyTo(charArray, 0);
            charArray[stringteste[0].Length] = 0;
            atribDS = H5S.create_simple(1, longsz);

            H5AttributeId Attribs3 = H5A.create(bvDset, "TimeStamp_meta", attrDt, atribDS);
            H5A.write(Attribs3, attrDt, new H5Array<byte>(charArray));


            H5S.close(atribDS);
            H5A.close(Attribs);
            H5A.close(Attribs2);
            H5A.close(Attribs3);
            H5D.close(bvDset);
            H5S.close(myDataSpace);
            bv.bvData = new List<OPC_UA>();
        }            

        H5G.close(datasetGroup);
        H5F.close(arquivoHDF5);
    }

Here is how the pandas reads and writes, being:
1- the value read by the pandas using the jupyter notebook (automatic processed value)

2- The actual value in the file

Andherearetheattributesthatpandascreatedautomatically(whichItriedtorecreateintheabovemethod)

    
asked by anonymous 14.06.2018 / 20:07

0 answers