track and return more of: formVersion, deviceId, userAgent

lib/data/briefcase.js

@@ -35,13 +35,14 @@ const pushPtr = (stack) => stack.push(ptr(stack));
 // given a row to write to, a decorated schema-field object, and metadata, writes
 // the metadata to the row and returns it.
 /* eslint-disable no-param-reassign */
-const writeMetadata = (out, idxs, submission, submitter, attachments, status) => {
+const writeMetadata = (out, idxs, submission, submitter, formVersion, attachments, status) => {
   out[idxs.submissionDate] = (new Date(submission.createdAt)).toISOString();
   out[idxs.key] = submission.instanceId;
   if (submitter != null) {
     out[idxs.submitterID] = submitter.id;
     out[idxs.submitterName] = submitter.displayName;
   }
+  out[idxs.formVersion] = formVersion;
   out[idxs.attachmentsPresent] = attachments.present || 0;
   out[idxs.attachmentsExpected] = attachments.expected || 0;
   if (status != null)
@@ -272,6 +273,7 @@ const streamBriefcaseCsvs = (inStream, inFields, xmlFormId, selectValues, decryp
     rootMeta.reviewState = rootHeader.length;          rootHeader.push('ReviewState');
     rootMeta.deviceID = rootHeader.length;             rootHeader.push('DeviceID');
     rootMeta.edits = rootHeader.length;                rootHeader.push('Edits');
+    rootMeta.formVersion = rootHeader.length;          rootHeader.push('FormVersion');
   }
 
   // then set up our main transform stream to handle incoming database records. it doesn't
@@ -303,13 +305,13 @@ const streamBriefcaseCsvs = (inStream, inFields, xmlFormId, selectValues, decryp
           (xml === null) ? 'not decrypted' : null; // eslint-disable-line indent
         if (status != null) {
           const result = new Array(rootHeader.length);
-          writeMetadata(result, rootMeta, submission, submission.aux.submitter, submission.aux.attachment, status);
+          writeMetadata(result, rootMeta, submission, submission.aux.submitter, submission.aux.exports.formVersion, submission.aux.attachment, status);
           return done(null, result);
         }
 
         // write the root row we get back from parsing the xml.
         processRow(xml, submission.instanceId, fields, rootHeader, selectValues).then((result) => {
-          writeMetadata(result, rootMeta, submission, submission.aux.submitter, submission.aux.attachment);
+          writeMetadata(result, rootMeta, submission, submission.aux.submitter, submission.aux.exports.formVersion, submission.aux.attachment);
           done(null, result);
         }, done); // pass through errors.
       } catch (ex) { done(ex); }

lib/data/odata.js

@@ -149,7 +149,8 @@ const submissionToOData = (fields, table, submission, options = {}) => new Promi
         reviewState: submission.reviewState || null,
         deviceId: submission.deviceId || null,
         // because of how we compute this value we don't need to default it to 0:
-        edits: submission.aux.edit.count
+        edits: submission.aux.edit.count,
+        formVersion: submission.aux.exports.formVersion
       }
     });
 

lib/formats/odata.js

@@ -143,6 +143,7 @@ const edmxTemplater = template(`<?xml version="1.0" encoding="UTF-8"?>
         <Property Name="reviewState" Type="org.opendatakit.submission.ReviewState"/>
         <Property Name="deviceId" Type="Edm.String"/>
         <Property Name="edits" Type="Edm.Int64"/>
+        <Property Name="formVersion" Type="Edm.String"/>
       </ComplexType>
       <EnumType Name="Status">
         <Member Name="notDecrypted"/>

lib/model/frames/submission.js

@@ -91,13 +91,19 @@ Submission.Def = Frame.define(
   'signature',                          'createdAt',    readable,
   'instanceName', readable,             'instanceId',   readable,
   'current',      readable,             'xml',
+  'root',                               'deviceId',     readable,
+  'userAgent',    readable,
   embedded('submitter')
 );
 
+Submission.Extended = Frame.define('formVersion', readable);
+
 Submission.Xml = Frame.define(table('submission_defs', 'xml'), 'xml');
 
 Submission.Encryption = Frame.define(into('encryption'), 'encHasData', 'encData', 'encIndex',  'encKeyId');
 
+Submission.Exports = Frame.define(into('exports'), 'formVersion');
+
 
 module.exports = { Submission };
 

lib/model/migrations/20211109-01-add-user-agent-to-submissions.js

@@ -0,0 +1,28 @@
+// Copyright 2021 ODK Central Developers
+// See the NOTICE file at the top-level directory of this distribution and at
+// https://github.com/getodk/central-backend/blob/master/NOTICE.
+// This file is part of ODK Central. It is subject to the license terms in
+// the LICENSE file found in the top-level directory of this distribution and at
+// https://www.apache.org/licenses/LICENSE-2.0. No part of ODK Central,
+// including this file, may be copied, modified, propagated, or distributed
+// except according to the terms contained in the LICENSE file.
+
+const up = async (db) => {
+  await db.schema.table('submission_defs', (sds) => {
+    sds.string('userAgent');
+    sds.string('deviceId');
+  });
+
+  await db.raw(`update submission_defs set "deviceId"=submissions."deviceId"
+    from submissions
+    where submissions.id=submission_defs."submissionId"
+      and submission_defs.id in (select min(id) from submission_defs group by "submissionId")`);
+};
+
+const down = (db) => db.schema.table('submission_defs', (sds) => {
+  sds.dropColumn('userAgent');
+  sds.dropColumn('deviceId');
+});
+
+module.exports = { up, down };
+

lib/model/migrations/20211114-01-flag-initial-submission-def.js

@@ -0,0 +1,23 @@
+// Copyright 2021 ODK Central Developers
+// See the NOTICE file at the top-level directory of this distribution and at
+// https://github.com/getodk/central-backend/blob/master/NOTICE.
+// This file is part of ODK Central. It is subject to the license terms in
+// the LICENSE file found in the top-level directory of this distribution and at
+// https://www.apache.org/licenses/LICENSE-2.0. No part of ODK Central,
+// including this file, may be copied, modified, propagated, or distributed
+// except according to the terms contained in the LICENSE file.
+
+const up = async (db) => {
+  await db.schema.table('submission_defs', (sds) => {
+    sds.boolean('root');
+  });
+  await db.raw(`update submission_defs set root=true
+  where id in (select min(id) from submission_defs group by "submissionId")`);
+};
+
+const down = (db) => db.schema.table('submission_defs', (sds) => {
+  sds.dropColumn('root');
+});
+
+module.exports = { up, down };
+

lib/model/query/submissions.js

@@ -20,18 +20,19 @@ const Problem = require('../../util/problem');
 ////////////////////////////////////////////////////////////////////////////////
 // SUBMISSION CREATE
 
-const _defInsert = (id, partial, formDefId, actorId) => sql`insert into submission_defs ("submissionId", xml, "formDefId", "instanceId", "instanceName", "submitterId", "localKey", "encDataAttachmentName", "signature", "createdAt", current)
-  values (${id}, ${sql.binary(partial.xml)}, ${formDefId}, ${partial.instanceId}, ${partial.def.instanceName}, ${actorId}, ${partial.def.localKey}, ${partial.def.encDataAttachmentName}, ${partial.def.signature}, clock_timestamp(), true)
+const _defInsert = (id, partial, formDefId, actorId, root, deviceId, userAgent) => sql`insert into submission_defs ("submissionId", xml, "formDefId", "instanceId", "instanceName", "submitterId", "localKey", "encDataAttachmentName", "signature", "createdAt", root, current, "deviceId", "userAgent")
+  values (${id}, ${sql.binary(partial.xml)}, ${formDefId}, ${partial.instanceId}, ${partial.def.instanceName}, ${actorId}, ${partial.def.localKey}, ${partial.def.encDataAttachmentName}, ${partial.def.signature}, clock_timestamp(), ${root}, true, ${deviceId}, ${userAgent})
   returning *`;
 const nextval = sql`nextval(pg_get_serial_sequence('submissions', 'id'))`;
 
 // creates both the submission and its initial submission def in one go.
-const createNew = (partial, form, deviceIdIn = null) => ({ one, context }) => {
+const createNew = (partial, form, deviceIdIn = null, userAgentIn = null) => ({ one, context }) => {
   const actorId = context.auth.actor.map((actor) => actor.id).orNull();
   const deviceId = blankStringToNull(deviceIdIn);
+  const userAgent = blankStringToNull(userAgentIn);
 
   return one(sql`
-with def as (${_defInsert(nextval, partial, form.def.id, actorId)}),
+with def as (${_defInsert(nextval, partial, form.def.id, actorId, true, deviceId, userAgent)}),
 ins as (insert into submissions (id, "formId", "instanceId", "submitterId", "deviceId", draft, "createdAt")
   select def."submissionId", ${form.id}, ${partial.instanceId}, def."submitterId", ${deviceId}, ${form.def.publishedAt == null}, def."createdAt" from def
   returning submissions.*)
@@ -56,15 +57,17 @@ createNew.audit = (submission, _, form) => (log) =>
   log('submission.create', form, { submissionId: submission.id, instanceId: submission.instanceId });
 createNew.audit.withResult = true;
 
-const createVersion = (partial, deprecated, form) => ({ one, context }) => {
+const createVersion = (partial, deprecated, form, deviceIdIn = null, userAgentIn = null) => ({ one, context }) => {
   const actorId = context.auth.actor.map((actor) => actor.id).orNull();
+  const deviceId = blankStringToNull(deviceIdIn);
+  const userAgent = blankStringToNull(userAgentIn);
 
   // we already do transactions but it just feels nice to have the cte do it all at once.
   return one(sql`
 with logical as (update submissions set "reviewState"='edited', "updatedAt"=clock_timestamp()
   where id=${deprecated.submissionId})
 , upd as (update submission_defs set current=false where "submissionId"=${deprecated.submissionId})
-${_defInsert(deprecated.submissionId, partial, form.def.id, actorId)}`)
+${_defInsert(deprecated.submissionId, partial, form.def.id, actorId, null, deviceId, userAgent)}`)
     // TODO/HACK: lame that we are reconstructing this this way.
     .then((saved) => new Submission({ instanceId: partial.instanceId },
       { def: new Submission.Def(saved), xml: new Submission.Xml({ xml: partial.xml }) }));
@@ -178,12 +181,15 @@ inner join submissions on submissions.id = submission_defs."submissionId" and su
 where submission_defs.id=${submissionDefId}`)
   .then(map(construct(Submission.Def)));
 
-const _getDef = extender(Submission.Def)(Actor.into('submitter'))((fields, extend, options, formId, draft) => sql`
+const _getDef = extender(Submission.Def)(Actor.into('submitter'), Submission.Extended)((fields, extend, options, formId, draft) => sql`
 select ${fields} from submission_defs
 inner join
   (select id, "instanceId" from submissions where "formId"=${formId} and draft=${draft} and "deletedAt" is null)
   as submissions on submissions.id=submission_defs."submissionId"
-${extend|| sql`left outer join actors on actors.id=submission_defs."submitterId"`}
+${extend|| sql`
+  left outer join actors on actors.id=submission_defs."submitterId"
+  inner join (select id, version as "formVersion" from form_defs) as fds
+    on fds.id=submission_defs."formDefId"`}
 where ${equals(options.condition)}
 order by submission_defs.id desc`);
 
@@ -211,7 +217,7 @@ where "formId"=${formId} and draft=${draft} and submissions."deletedAt" is null`
 
 const _exportUnjoiner = unjoiner(Submission, Submission.Def, Submission.Xml, Submission.Encryption,
   Actor.alias('actors', 'submitter'), Frame.define(table('attachments'), 'present', 'expected'),
-  Frame.define(table('edits'), 'count'));
+  Frame.define(table('edits'), 'count'), Submission.Exports); // TODO: figure out how to combine some of these
 
 // TODO: this is a terrible hack to add some logic to one of our select fields. this is
 // the /only/ place we need to do this in the entire codebase right now. so for now
@@ -241,6 +247,10 @@ left outer join
     from submission_attachments
     group by "submissionDefId") as attachments
   on attachments."submissionDefId"=submission_defs.id
+inner join (select "formDefId", "submissionId" from submission_defs where root is true) as roots
+  on roots."submissionId"=submission_defs."submissionId"
+inner join (select id, version as "formVersion" from form_defs) as fds
+  on fds.id=roots."formDefId"
 ${encrypted ? sql`
 left outer join (select id, "keyId" as "encKeyId" from form_defs) as form_defs on form_defs.id=submission_defs."formDefId"
 left outer join (select id, content as "encData" from blobs) as blobs on blobs.id=submission_attachments."blobId"`

lib/resources/submissions.js

@@ -68,7 +68,7 @@ module.exports = (service, endpoint) => {
         .then(always({ code: 204, body: '' }))));
 
     // Nonstandard REST; OpenRosa-specific API.
-    service.post(path, multipart.any(), endpoint.openRosa(({ Forms, Submissions, SubmissionAttachments }, { params, files, auth, query }) =>
+    service.post(path, multipart.any(), endpoint.openRosa(({ Forms, Submissions, SubmissionAttachments }, { params, files, auth, query, headers }) =>
       Submission.fromXml(findMultipart(files).buffer)
         .then((partial) => getForm(auth, params, partial.xmlFormId, Forms, partial.def.version)
           .catch(forceAuthFailed)
@@ -111,7 +111,7 @@ module.exports = (service, endpoint) => {
                   }
                   // NEW SUBMISSION REQUEST: create a new submission and attachments.
                   return Promise.all([
-                    Submissions.createNew(partial, form, query.deviceID),
+                    Submissions.createNew(partial, form, query.deviceID, headers['user-agent']),
                     Forms.getBinaryFields(form.def.id)
                   ]).then(([ saved, binaryFields ]) => SubmissionAttachments.create(saved, form, binaryFields, files));
                 });
@@ -162,7 +162,7 @@ module.exports = (service, endpoint) => {
   // and repeated for draft/nondraft.
 
   const restSubmission = (base, draft, getForm) => {
-    service.post(`${base}/submissions`, endpoint(({ Forms, Submissions, SubmissionAttachments }, { params, auth, query }, request) =>
+    service.post(`${base}/submissions`, endpoint(({ Forms, Submissions, SubmissionAttachments }, { params, auth, query, headers }, request) =>
       Submission.fromXml(request)
         .then((partial) => getForm(params, Forms, partial.def.version)
           .then((form) => auth.canOrReject('submission.create', form))
@@ -171,15 +171,15 @@ module.exports = (service, endpoint) => {
               return reject(Problem.user.unexpectedValue({ field: 'form id', value: partial.xmlFormId, reason: 'did not match the form ID in the URL' }));
 
             return Promise.all([
-              Submissions.createNew(partial, form, query.deviceID),
+              Submissions.createNew(partial, form, query.deviceID, headers['user-agent']),
               Forms.getBinaryFields(form.def.id)
             ])
               .then(([ submission, binaryFields ]) =>
                 SubmissionAttachments.create(submission, form, binaryFields)
                   .then(always(submission)));
           }))));
 
-    service.put(`${base}/submissions/:instanceId`, endpoint(({ Forms, Submissions, SubmissionAttachments }, { params, auth }, request) =>
+    service.put(`${base}/submissions/:instanceId`, endpoint(({ Forms, Submissions, SubmissionAttachments }, { params, auth, query, headers }, request) =>
       Submission.fromXml(request).then((partial) => {
         if (partial.xmlFormId !== params.formId)
           return reject(Problem.user.unexpectedValue({ field: 'form id', value: partial.xmlFormId, reason: 'did not match the form ID in the URL' }));
@@ -196,7 +196,7 @@ module.exports = (service, endpoint) => {
             .then(getOrNotFound) // this request exists just to check existence and fail the whole request.
         ])
           .then(([ deprecated, form ]) => Promise.all([
-            Submissions.createVersion(partial, deprecated, form),
+            Submissions.createVersion(partial, deprecated, form, query.deviceID, headers['user-agent']),
             Forms.getBinaryFields(form.def.id),
             SubmissionAttachments.getForFormAndInstanceId(form.id, deprecatedId, draft),
           ])

test/assertions.js

@@ -250,13 +250,13 @@ should.Assertion.add('SimpleCsv', function() {
 
   const csv = this.obj.split('\n').map((row) => row.split(','));
   csv.length.should.equal(5); // header + 3 data rows + newline
-  csv[0].should.eql([ 'SubmissionDate', 'meta-instanceID', 'name', 'age', 'KEY', 'SubmitterID', 'SubmitterName', 'AttachmentsPresent', 'AttachmentsExpected', 'Status', 'ReviewState', 'DeviceID', 'Edits' ]);
+  csv[0].should.eql([ 'SubmissionDate', 'meta-instanceID', 'name', 'age', 'KEY', 'SubmitterID', 'SubmitterName', 'AttachmentsPresent', 'AttachmentsExpected', 'Status', 'ReviewState', 'DeviceID', 'Edits', 'FormVersion' ]);
   csv[1].shift().should.be.an.recentIsoDate();
-  csv[1].should.eql([ 'three','Chelsea','38','three','5','Alice','0','0','','','','0' ]);
+  csv[1].should.eql([ 'three','Chelsea','38','three','5','Alice','0','0','','','','0', '' ]);
   csv[2].shift().should.be.an.recentIsoDate();
-  csv[2].should.eql([ 'two','Bob','34','two','5','Alice','0','0','','','','0' ]);
+  csv[2].should.eql([ 'two','Bob','34','two','5','Alice','0','0','','','','0', '' ]);
   csv[3].shift().should.be.an.recentIsoDate();
-  csv[3].should.eql([ 'one','Alice','30','one','5','Alice','0','0','','','','0' ]);
+  csv[3].should.eql([ 'one','Alice','30','one','5','Alice','0','0','','','','0', '' ]);
   csv[4].should.eql([ '' ]);
 });
 
@@ -265,12 +265,15 @@ should.Assertion.add('EncryptedSimpleCsv', function() {
 
   const csv = this.obj.split('\n').map((row) => row.split(','));
   csv.length.should.equal(5); // header + 3 data rows + newline
-  csv[0].should.eql([ 'SubmissionDate', 'meta-instanceID', 'name', 'age', 'KEY', 'SubmitterID', 'SubmitterName', 'AttachmentsPresent', 'AttachmentsExpected', 'Status', 'ReviewState', 'DeviceID', 'Edits' ]);
+  csv[0].should.eql([ 'SubmissionDate', 'meta-instanceID', 'name', 'age', 'KEY', 'SubmitterID', 'SubmitterName', 'AttachmentsPresent', 'AttachmentsExpected', 'Status', 'ReviewState', 'DeviceID', 'Edits', 'FormVersion' ]);
   csv[1].shift().should.be.an.recentIsoDate();
+  csv[1].pop().should.match(/^\[encrypted:........\]$/);
   csv[1].should.eql([ 'three','Chelsea','38','three','5','Alice','1','1','','','','0' ]);
   csv[2].shift().should.be.an.recentIsoDate();
+  csv[2].pop().should.match(/^\[encrypted:........\]$/);
   csv[2].should.eql([ 'two','Bob','34','two','5','Alice','1','1','','','','0' ]);
   csv[3].shift().should.be.an.recentIsoDate();
+  csv[3].pop().should.match(/^\[encrypted:........\]$/);
   csv[3].should.eql([ 'one','Alice','30','one','5','Alice','1','1','','','','0' ]);
   csv[4].should.eql([ '' ]);
 });