Import of data

There is NodeJS library for importing data to MyDataSpace.

At the moment this is the only way to import large amounts of data into MyDataSpace. In the future, it will be possible to import data from files using the Administrative Panel.

Below is a step-by-step description of the process of importing Geonames postal codes.

1. Create root and element for the data

Root — is a root element of the data tree.

For convenience, create a child element postal-codes and put the data from the file.

https://myda.space

2. Create app

To allow our NodeJS-app save data to MyDataSpace, we should grant permissions to it over app.

Create new app and allow access from any IP addresses (URLs & IPs should be 0.0.0.0).

https://myda.space

3. Download data

Download and extract files with data.

4. Create NodeJS app

mkdir import-geonames-data
cd import-geonames-data
npm init
npm install mydataspace --save
touch app.js

app.js (Do not forget to specify ACCESS_TOKEN, APP_API_KEY, FILE_NAME, ROOT):

// Your access key
const ACCESS_TOKEN = '***********';

// Your app API key
const APP_API_KEY = '***********';

// Path to the file with data
const FILE_NAME = '/path/to/RU.txt';

const ROOT = 'geonames-demo';

const crypto = require('crypto');
const mds = require('mydataspace');

process.env['NODE_TLS_REJECT_UNAUTHORIZED'] = '0';

const myda = new mds.Myda({
  root: ROOT,
  permission: ROOT,
  clientId: APP_API_KEY
});

var lineReader;

var block = [];
var linesRead = 0;


/**
 * Connect to MyDataSpace and start importing data.
 */
myda.connect().then(() => myda.loginByToken(ACCESS_TOKEN)).then(function() {
  console.log('--- Logged in ---');
  lineReader = require('readline').createInterface({
    input: require('fs').createReadStream(FILE_NAME)
  });
  lineReader.on('line', parseLineAndAddToBlock);
  lineReader.on('pause', sendBlockToServer);
  lineReader.on('end', sendBlockToServer.bind(null, true));
}).catch(function(err) {
	console.log('LOGIN ERROR:');
	console.log(err);
});

function parseLineAndAddToBlock(line) {
  linesRead++;
  if (linesRead % 100 === 0) console.log('    parse ' + linesRead);
  const parts = line.split('\t');
  const suffix = crypto.createHash('md5').update(line).digest('hex').substring(0, 7);
  block.push({
    root: ROOT,
    path: 'postal-codes/' + parts[0] + '-' + parts[1] + '-' + suffix,
    fields: [
        { name: 'code',       value: parts[1],  type: 's' },
        { name: 'placeName',  value: parts[2],  type: 's' },
        { name: 'adminName1', value: parts[3],  type: 's' },
        { name: 'adminCode1', value: parts[4],  type: 's' },
        { name: 'adminName2', value: parts[5],  type: 's' },
        { name: 'adminCode2', value: parts[6],  type: 's' },
        { name: 'adminName3', value: parts[7],  type: 's' },
        { name: 'adminCode3', value: parts[8],  type: 's' },
        { name: 'latitude',   value: parts[9],  type: 's' },
        { name: 'longitude',  value: parts[10], type: 's' },
        { name: 'accuracy',   value: parts[11], type: 's' }
    ]
  });
  if (block.length >= 1000) {
    lineReader.pause();
  }
}

function nextLineOrExit(isLastBlock, isError) {
  if (isError) {
    if (isLastBlock) {
      console.log('*** Importing finished with error! ***');
      process.exit();
    } else {
      console.log('--- Block sent with errors. Reading new block... ---');
      lineReader.resume();
    }
  } else {
    if (isLastBlock) {
      console.log('*** Importing finished! ***');
      process.exit();
    } else {
      console.log('---Block sent successful. Reading new block... ---');
      lineReader.resume();
    }
  }
}

function sendBlockToServer(isLastBlock) {
  console.log('New block of ' + block.length + ' recoreds received. Sending it to server...');
  var blockForSend = block;
  block = [];
  myda.request('entities.create', blockForSend, function(data) {
    nextLineOrExit(isLastBlock, false);
  }, function(err) {
    console.log('ERROR:');
  	console.log(err);
    console.log('INNER ERRORS:');
    (Array.isArray(err) ? err : []).forEach(x => console.log(x.errors));
    nextLineOrExit(isLastBlock, true);
  });
}

You can take the source code of the example on GitHub.

5. Run app

node app

Importing large amounts of data is a long process. Be patient

Here you can see the result of the program.