From 5805869d53a975dfbc35df1a3a46fb9f946cc5e8 Mon Sep 17 00:00:00 2001 From: Jeremy LaCivita Date: Tue, 5 Dec 2023 22:03:31 -0500 Subject: [PATCH 1/2] feat: Protocol Requirements --- src/openrpc/index.mjs | 13 ++++--- src/shared/json-schema.mjs | 8 +++-- src/shared/modules.mjs | 70 +++++++++++++++++++++++++++++++------- 3 files changed, 71 insertions(+), 20 deletions(-) diff --git a/src/openrpc/index.mjs b/src/openrpc/index.mjs index e0819b7f..a1c000f7 100644 --- a/src/openrpc/index.mjs +++ b/src/openrpc/index.mjs @@ -45,12 +45,17 @@ const run = async ({ Object.entries(sharedSchemas).forEach(([path, schema]) => { const json = JSON.parse(schema) const id = json.$id - sharedSchemas[id] = json - delete sharedSchemas[path] + if (id) { + sharedSchemas[id] = json + delete sharedSchemas[path] + } + else { + sharedSchemas[path] = json + } }) const moduleList = input ? await readDir(path.join(input, 'openrpc'), { recursive: true }) : [] - const modules = await readFiles(moduleList, path.join(input, 'openrpc')) + const modules = await readFiles(moduleList, path.resolve('..')) const descriptionsList = input ? await readDir(path.join(input, 'descriptions'), { recursive: true }) : [] const markdown = await readFiles(descriptionsList, path.join(input, 'descriptions')) @@ -94,7 +99,7 @@ const run = async ({ json.components && Object.assign(openrpc.components.schemas, json.components.schemas) // add externally referenced schemas that are in our shared schemas path - openrpc = addExternalSchemas(openrpc, sharedSchemas) + openrpc = addExternalSchemas(openrpc, sharedSchemas, path.dirname(key)) modules[key] = JSON.stringify(json, null, '\t') diff --git a/src/shared/json-schema.mjs b/src/shared/json-schema.mjs index e7bb1d86..19687205 100644 --- a/src/shared/json-schema.mjs +++ b/src/shared/json-schema.mjs @@ -18,6 +18,8 @@ import deepmerge from 'deepmerge' import crocks from 'crocks' +import path from 'path' + const { setPath, getPathOr } = crocks const isNull = schema => { @@ -103,16 +105,16 @@ const replaceUri = (existing, replacement, schema) => { } } -const replaceRef = (existing, replacement, schema) => { +const replaceRef = (existing, replacement, schema, dir = '') => { if (schema) { if (schema.hasOwnProperty('$ref') && (typeof schema['$ref'] === 'string')) { - if (schema['$ref'] === existing) { + if (path.join(dir, schema['$ref']) === existing) { schema['$ref'] = replacement } } else if (typeof schema === 'object') { Object.keys(schema).forEach(key => { - replaceRef(existing, replacement, schema[key]) + replaceRef(existing, replacement, schema[key], dir) }) } } diff --git a/src/shared/modules.mjs b/src/shared/modules.mjs index 927092c2..1795e2d7 100644 --- a/src/shared/modules.mjs +++ b/src/shared/modules.mjs @@ -31,6 +31,7 @@ import predicates from 'crocks/predicates/index.js' import { getExternalSchemaPaths, isDefinitionReferencedBySchema, isNull, localizeDependencies, isSchema, getLocalSchemaPaths, replaceRef } from './json-schema.mjs' import { getPath as getRefDefinition } from './json-schema.mjs' const { isObject, isArray, propEq, pathSatisfies, hasProp, propSatisfies } = predicates +import path from 'path' // util for visually debugging crocks ADTs const inspector = obj => { @@ -630,7 +631,8 @@ const createResponseFromProvider = (provider, type, json) => { schema: { allOf: [ { - "$ref": "https://meta.comcast.com/firebolt/types#/definitions/ProviderResponse" // use this schema for both Errors and Results + "$ref": "#/x-schemas/Types/ProviderResponse" // use this schema for both Errors and Results +// "$ref": "https://meta.comcast.com/firebolt/types#/definitions/ProviderResponse" // use this schema for both Errors and Results }, { "type": "object", @@ -843,7 +845,8 @@ const generateEventListenResponse = json => { // only want or and xor here (might even remove xor) const anyOf = event.result.schema.oneOf || event.result.schema.anyOf const ref = { - "$ref": "https://meta.comcast.com/firebolt/types#/definitions/ListenResponse" + "$ref": "#/x-schemas/Types/ListenResponse" +// "$ref": "https://meta.comcast.com/firebolt/types#/definitions/ListenResponse" } if (anyOf) { @@ -1051,7 +1054,7 @@ const getExternalPath = (uri = '', schemas = {}) => { const [mainPath, subPath] = uri.split('#') const json = schemas[mainPath] || schemas[mainPath + '/'] - + // copy to avoid side effects let result @@ -1067,7 +1070,7 @@ const getExternalPath = (uri = '', schemas = {}) => { return result } -const getExternalSchemas = (json = {}, schemas = {}) => { +const getExternalSchemas = (json = {}, schemas = {}, dir = '') => { // make a copy for safety! json = JSON.parse(JSON.stringify(json)) @@ -1077,20 +1080,58 @@ const getExternalSchemas = (json = {}, schemas = {}) => { while (refs.length > 0) { for (let i=0; i { + external.pop() + const node = getPathOr(null, external, resolvedSchema) + const target = path.join(path.dirname(ref.split('#')[0]), node.$ref.split('#')[0]) + // console.log(target) + const modified = path.relative(dir, target) + // console.log('old: ' + node.$ref) + // console.log('rel: ' + dir) + // console.log('tar: ' + target) + // console.log('mod: ' + modified) + const hash = node.$ref.split('#')[1] + node.$ref = modified + '#' + hash//[modified].concat(node.$ref.split('#')).slice(1).join('#') + console.log('fin: ' + node.$ref) + }) + + const locals = getLocalSchemaPaths(resolvedSchema) + locals.forEach(local => { + local.pop() + const node = getPathOr(null, local, resolvedSchema) + console.log('loc0: ' + ref.split('#')[0]) + console.log('loc1: ' + node.$ref) + const target = path.join(path.dirname(ref.split('#')[0]), node.$ref.split('#')[0]) + const modified = path.relative(dir, ref.split('#')[0]) + console.log('loc2: ' + node.$ref) + console.log('loc3: ' + dir) + console.log('loc4: ' + target) + console.log('loc5: ' + modified) + const hash = node.$ref.split('#')[1] + node.$ref = modified + '#' + hash//[modified].concat(node.$ref.split('#')).slice(1).join('#') + console.log('loc!: ' + node.$ref) + }) + // replace the ref so we can recursively grab more refs if needed... - else if (path.length) { + if (refPath.length) { returnedSchemas[ref] = JSON.parse(JSON.stringify(resolvedSchema)) // use a copy, so we don't pollute the returned schemas - json = setPath(path, JSON.parse(JSON.stringify(resolvedSchema)), json) + json = setPath(refPath, JSON.parse(JSON.stringify(resolvedSchema)), json) } else { delete json['$ref'] @@ -1103,17 +1144,18 @@ const getExternalSchemas = (json = {}, schemas = {}) => { return returnedSchemas } -const addExternalSchemas = (json, sharedSchemas) => { +const addExternalSchemas = (json, sharedSchemas, dir) => { + console.log('addExternalSchemas') json = JSON.parse(JSON.stringify(json)) let searching = true while (searching) { searching = false - const externalSchemas = getExternalSchemas(json, sharedSchemas) + const externalSchemas = getExternalSchemas(json, sharedSchemas, dir) Object.entries(externalSchemas).forEach( ([name, schema]) => { const group = sharedSchemas[name.split('#')[0]].title - const id = sharedSchemas[name.split('#')[0]].$id + const id = sharedSchemas[name.split('#')[0]].$id || name.split('#')[0] const refs = getLocalSchemaPaths(schema) refs.forEach(ref => { ref.pop() // drop the actual '$ref' so we can modify it @@ -1127,6 +1169,7 @@ const addExternalSchemas = (json, sharedSchemas) => { return } searching = true + json['x-schemas'] = json['x-schemas'] || {} json['x-schemas'][group] = json['x-schemas'][group] || { uri: name.split("#")[0]} json['x-schemas'][group][name.split("/").pop()] = schema @@ -1134,8 +1177,9 @@ const addExternalSchemas = (json, sharedSchemas) => { //update references to external schemas to be local Object.keys(externalSchemas).forEach(ref => { + console.log('localizing: ' + ref) const group = sharedSchemas[ref.split('#')[0]].title - replaceRef(ref, `#/x-schemas/${group}/${ref.split("#").pop().substring('/definitions/'.length)}`, json) + replaceRef(ref, `#/x-schemas/${group}/${ref.split("#").pop().substring('/definitions/'.length)}`, json, dir) }) } From 51690c067297c27f82df0680ae1ec7c1c532cbea Mon Sep 17 00:00:00 2001 From: Jeremy LaCivita Date: Thu, 18 Jan 2024 15:58:44 -0500 Subject: [PATCH 2/2] feat: Support relative $refs and update 1.0 -> 2.0 script --- src/cli.mjs | 7 +- src/openrpc/index.mjs | 8 +-- src/shared/filesystem.mjs | 11 +-- src/shared/methods.mjs | 6 ++ src/update/index.mjs | 143 ++++++++++++++++++++++++++++++++++++++ 5 files changed, 165 insertions(+), 10 deletions(-) create mode 100644 src/shared/methods.mjs create mode 100644 src/update/index.mjs diff --git a/src/cli.mjs b/src/cli.mjs index 8e09f6c7..896dcac8 100755 --- a/src/cli.mjs +++ b/src/cli.mjs @@ -5,6 +5,7 @@ import sdk from './sdk/index.mjs' import docs from './docs/index.mjs' import openrpc from './openrpc/index.mjs' import validate from './validate/index.mjs' +import update from './update/index.mjs' import nopt from 'nopt' import path from 'path' @@ -62,6 +63,10 @@ else if (task === 'validate') { } else if (task === 'openrpc') { openrpc(parsedArgs).then(signOff) -} else { +} +else if (task === 'update') { + update(parsedArgs).then(signOff) +} +else { console.log("Invalid build type") } \ No newline at end of file diff --git a/src/openrpc/index.mjs b/src/openrpc/index.mjs index a1c000f7..f84e821a 100644 --- a/src/openrpc/index.mjs +++ b/src/openrpc/index.mjs @@ -29,8 +29,8 @@ const run = async ({ }) => { let openrpc = await readJson(template) - const sharedSchemaList = schemas ? (await Promise.all(schemas.map(d => readDir(d, { recursive: true })))).flat() : [] - const sharedSchemas = await readFiles(sharedSchemaList) + const sharedSchemaList = schemas ? (await Promise.all(schemas.map(d => readDir(d, { recursive: true, base: path.resolve('.') })))).flat() : [] + const sharedSchemas = await readFiles(sharedSchemaList, path.resolve('.')) try { const packageJson = await readJson(path.join(input, '..', 'package.json')) @@ -54,10 +54,10 @@ const run = async ({ } }) - const moduleList = input ? await readDir(path.join(input, 'openrpc'), { recursive: true }) : [] + const moduleList = input ? await readDir(path.join(input, 'openrpc'), { recursive: true, base: path.resolve('.') }) : [] const modules = await readFiles(moduleList, path.resolve('..')) - const descriptionsList = input ? await readDir(path.join(input, 'descriptions'), { recursive: true }) : [] + const descriptionsList = input ? await readDir(path.join(input, 'descriptions'), { recursive: true, base: path.resolve('.') }) : [] const markdown = await readFiles(descriptionsList, path.join(input, 'descriptions')) Object.keys(modules).forEach(key => { diff --git a/src/shared/filesystem.mjs b/src/shared/filesystem.mjs index 2c48e139..fbcb6cda 100644 --- a/src/shared/filesystem.mjs +++ b/src/shared/filesystem.mjs @@ -22,17 +22,18 @@ const readJson = ref => readFile(ref) const writeJson = (ref, json) => writeText(ref, JSON.stringify(json, null, '\t')) const readDir = async (ref, options) => { + + if (!options.base) { + options.base = path.join(ref, '..') + } + let i = 0 const isJustAFile = lstatSync(ref).isFile() const files = isJustAFile ? [ { name:'', isDirectory: _ => false } ] : await readdir(ref, { withFileTypes: true }) - const results = files.filter(file => !file.isDirectory()).map(file => path.join(ref, file.name)) + const results = files.filter(file => !file.isDirectory()).map(file => path.relative(options.base, path.join(ref, file.name))) - if (!options.base) { - options.base = path.join(ref, '..') - } if (options.recursive) { for (var index=files.length-1; index>=0; index--) { - if (files[index].isDirectory()) { results.push(...((await readDir(path.join(ref, files[index].name), options)))) } diff --git a/src/shared/methods.mjs b/src/shared/methods.mjs new file mode 100644 index 00000000..cb3fb97d --- /dev/null +++ b/src/shared/methods.mjs @@ -0,0 +1,6 @@ +const tag = (method, name) => method.tags.find(tag => tag.name === name) +export const capabilities = method => tag(method, 'capabilities') +export const provides = method => capabilities(method)['x-provides'] +export const pusher = method => capabilities(method)['x-push'] +export const notifier = method => method.tags.find(t => t.name === 'notifier') +export const event = method => tag(method, 'event') \ No newline at end of file diff --git a/src/update/index.mjs b/src/update/index.mjs new file mode 100644 index 00000000..9d53b87b --- /dev/null +++ b/src/update/index.mjs @@ -0,0 +1,143 @@ +/* + * Copyright 2021 Comcast Cable Communications Management, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ + +import { readJson, readFiles, readDir, writeJson, writeFiles } from "../shared/filesystem.mjs" +import { addExternalMarkdown, addExternalSchemas, fireboltize } from "../shared/modules.mjs" +import path from "path" +import { logHeader, logSuccess } from "../shared/io.mjs" +import { capabilities, event, provides, pusher } from "../shared/methods.mjs" +import { getPath } from '../shared/json-schema.mjs' + +const run = async ({ + input: input, + output: output +}) => { + + logHeader(`Upgrading modules to latest Firebolt OpenRPC schema`) + + console.dir(input) + const moduleList = input ? await readDir(path.join(input, 'openrpc'), { recursive: true }) : [] + console.dir(moduleList) + const modules = await readFiles(moduleList, path.resolve('.') + '/.') + console.dir(Object.keys(modules)) + + Object.keys(modules).forEach(key => { + let json = JSON.parse(modules[key]) + + // Do the firebolt API magic + update(json) + + modules[key] = JSON.stringify(json, null, '\t') + + logSuccess(`Updated the ${json.info.title} module.`) + }) + + await writeFiles(modules) +console.log(Object.keys(modules)) + + console.log() + logSuccess(`Wrote file ${path.relative('.', output)}`) + + return Promise.resolve() +} + +function update(json) { + json.methods = json.methods.map(method => { + // update providers + if (provides(method)) { + // handle Provider Interfaces + if (method.name.startsWith('onRequest')) { + // simplify name + method.name = method.name.charAt(9).toLowerCase() + method.name.substr(10) + + // move params out of custom extension, and unwrap them into individual parameters + method.params = [] + const request = getPath(method.result.schema.$ref, json) + const params = getPath((request.allOf ? request.allOf[1] : request).properties.parameters.$ref, json) + + // add required params first, in order listed + params.required && params.required.forEach(p => { + method.params.push({ + name: p, + required: true, + schema: params.properties[p] + }) + delete params.properties[p] + }) + + // add unrequired params in arbitrary order... (there's currently no provider method method with more than one unrequired param) + Object.keys(params.properties).forEach(p => { + method.params.push({ + name: p, + required: false, + schema: params[p] + }) + delete params.properties[p] + }) + + + // move result out of custom extension + method.result = { + name: 'result', + schema: event(method)['x-response'] + } + + // fix example pairings + method.examples.forEach((example, i) => { + example.params = Object.entries(example.result.value.parameters).map(entry => ({ + name: entry[0], + value: entry[1] + })) + const result = method.result.schema.examples ? method.result.schema.examples[Math.min(i, method.result.schema.examples.length-1)] : getPath(method.result.schema.$ref, json).examples[0] + example.result = { + "name": "result", + "value": result + } + }) + + // delete examples, TODO: this needs to go into the method pairing examples... + delete method.result.schema.examples + + // TODO handle x-error + + method.tags = method.tags.filter(tag => (tag.name !== "event" && tag.name !== "rpc-only")) + } + } + else if (event(method)) { + console.dir(method.name) + // store the subscriber name in the x-event extension + event(method)['x-event'] = json.info.title + '.' + method.name + + // simplify name + method.name = method.name.charAt(2).toLowerCase() + method.name.substr(3) + // move the result into the single param + method.params = [ + method.result + ] + + // rename the event tag to notifier + event(method).name = "notifier" + + // remove the result, since this is a notification + delete method.result + } + return method + }) +} + +export default run \ No newline at end of file