Skip to content

Commit 6a8e09f

Browse files
webprofusion-chriscwebprofusion-chrisc
committed
Implement config-based OCPI import via web app queued imports, improve provider/import admin
1 parent 77ce615 commit 6a8e09f

22 files changed

Lines changed: 1719 additions & 107 deletions

‎API/OCM.Net/OCM.API.Core/Common/DataProviderManager.cs‎

Lines changed: 29 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
using System;
22
using System.Linq;
3+
using Microsoft.EntityFrameworkCore;
34

45
namespace OCM.API.Common
56
{
@@ -46,10 +47,21 @@ public Model.DataProvider CreateOCPIDataProvider(string title, string websiteUrl
4647

4748
public Model.DataProvider GetDataProviderByAgreementId(int agreementId)
4849
{
49-
var dataProvider = dataModel.DataProviders.FirstOrDefault(dp => dp.DataSharingAgreementId == agreementId);
50+
var dataProvider = dataModel.DataProviders
51+
.Include(dp => dp.DataProviderStatusType)
52+
.FirstOrDefault(dp => dp.DataSharingAgreementId == agreementId);
5053
return Model.Extensions.DataProvider.FromDataModel(dataProvider);
5154
}
5255

56+
public System.Collections.Generic.List<Model.DataProviderStatusType> GetDataProviderStatusTypes()
57+
{
58+
return dataModel.DataProviderStatusTypes
59+
.OrderBy(status => status.Title)
60+
.ToList()
61+
.Select(Model.Extensions.DataProviderStatusType.FromDataModel)
62+
.ToList();
63+
}
64+
5365
public string GetImportConfigByAgreementId(int agreementId)
5466
{
5567
return dataModel.DataProviders
@@ -58,9 +70,22 @@ public string GetImportConfigByAgreementId(int agreementId)
5870
.FirstOrDefault();
5971
}
6072

73+
public System.Collections.Generic.List<int> GetApprovedImportAgreementIds()
74+
{
75+
return dataModel.DataProviders
76+
.Where(dp => dp.IsApprovedImport == true
77+
&& dp.DataSharingAgreementId.HasValue
78+
&& !string.IsNullOrWhiteSpace(dp.ImportConfig))
79+
.Select(dp => dp.DataSharingAgreementId.Value)
80+
.Distinct()
81+
.ToList();
82+
}
83+
6184
public void SetImportApprovalStatus(int dataProviderId, bool isApproved)
6285
{
63-
var dataProvider = dataModel.DataProviders.FirstOrDefault(dp => dp.Id == dataProviderId);
86+
var dataProvider = dataModel.DataProviders
87+
.Include(dp => dp.DataProviderStatusType)
88+
.FirstOrDefault(dp => dp.Id == dataProviderId);
6489
if (dataProvider == null)
6590
{
6691
return;
@@ -70,7 +95,7 @@ public void SetImportApprovalStatus(int dataProviderId, bool isApproved)
7095
dataModel.SaveChanges();
7196
}
7297

73-
public Model.DataProvider UpdateOCPIDataProvider(int dataProviderId, string title, string websiteUrl, string license, bool isOpenDataLicensed, string ocpiConfigJson, int updatedByUserId)
98+
public Model.DataProvider UpdateOCPIDataProvider(int dataProviderId, string title, string websiteUrl, string license, bool isOpenDataLicensed, string ocpiConfigJson, int? dataProviderStatusTypeId, int updatedByUserId)
7499
{
75100
var dataProvider = dataModel.DataProviders.FirstOrDefault(dp => dp.Id == dataProviderId);
76101
if (dataProvider == null)
@@ -84,6 +109,7 @@ public Model.DataProvider UpdateOCPIDataProvider(int dataProviderId, string titl
84109
dataProvider.IsOpenDataLicensed = isOpenDataLicensed;
85110
dataProvider.ImportConfig = ocpiConfigJson;
86111
dataProvider.Comments = "OCPI with Data Sharing Agreement";
112+
dataProvider.DataProviderStatusTypeId = dataProviderStatusTypeId;
87113

88114
dataModel.SaveChanges();
89115

‎API/OCM.Net/OCM.API.Core/Common/DataSharingAgreementManager.cs‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -79,5 +79,30 @@ public Model.DataSharingAgreement UpdateAgreement(Model.DataSharingAgreement agr
7979

8080
return Model.Extensions.DataSharingAgreement.FromDataModel(item);
8181
}
82+
83+
public bool DeleteAgreement(int id, int deletedByUserId)
84+
{
85+
var item = dataModel.DataSharingAgreements.FirstOrDefault(a => a.Id == id);
86+
if (item == null)
87+
{
88+
return false;
89+
}
90+
91+
var linkedProviders = dataModel.DataProviders.Where(dp => dp.DataSharingAgreementId == id).ToList();
92+
foreach (var provider in linkedProviders)
93+
{
94+
provider.DataSharingAgreementId = null;
95+
}
96+
97+
dataModel.DataSharingAgreements.Remove(item);
98+
dataModel.SaveChanges();
99+
100+
var user = new UserManager().GetUser(deletedByUserId);
101+
AuditLogManager.Log(user, AuditEventType.DeletedItem,
102+
"{EntityType:\"DataSharingAgreement\",EntityID:" + id + "}",
103+
$"Deleted data sharing agreement for {item.CompanyName}");
104+
105+
return true;
106+
}
82107
}
83108
}

‎Import/OCM.Import.Common/ImportManager.cs‎

Lines changed: 81 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -59,6 +59,14 @@ public class ImportReport
5959
{
6060
public BaseImportProvider ProviderDetails { get; set; }
6161

62+
public bool IsPreviewMode { get; set; }
63+
64+
public int SourceItemCount { get; set; }
65+
66+
public int ProcessedSourceItemCount { get; set; }
67+
68+
public int PreviewItemLimit { get; set; }
69+
6270
public List<ImportItem> ImportItems { get; set; }
6371

6472
public List<ChargePoint> Added { get; set; }
@@ -102,6 +110,10 @@ public class ImportProcessSettings
102110
public bool PerformDeduplication { get; set; } = true;
103111
public string ProviderName { get; set; }
104112

113+
public bool IsPreviewMode { get; set; } = false;
114+
115+
public int PreviewItemLimit { get; set; } = 100;
116+
105117
public Dictionary<string, string> Credentials { get; set; }
106118
}
107119

@@ -261,6 +273,12 @@ void AddConfiguredProviders(IEnumerable<IImportProvider> configuredProviders)
261273
}
262274

263275
public async Task<bool> PerformImportProcessing(ImportProcessSettings settings, IImportProvider importProvider = null)
276+
{
277+
var result = await PerformImportProcessingWithReport(settings, importProvider);
278+
return result?.IsSuccess == true;
279+
}
280+
281+
public async Task<ImportReport> PerformImportProcessingWithReport(ImportProcessSettings settings, IImportProvider importProvider = null)
264282
{
265283

266284
var credentials = GetAPISessionCredentials("System", settings.Credentials["IMPORT-ocm-system"]);
@@ -273,7 +291,7 @@ public async Task<bool> PerformImportProcessing(ImportProcessSettings settings,
273291

274292
Log(result.Log);
275293

276-
return result.IsSuccess;
294+
return result;
277295
}
278296
else
279297
{
@@ -289,7 +307,7 @@ public async Task<bool> PerformImportProcessing(ImportProcessSettings settings,
289307

290308
Log(result.Log);
291309

292-
return result.IsSuccess;
310+
return result;
293311

294312
}
295313
}
@@ -300,7 +318,11 @@ public async Task<bool> PerformImportProcessing(ImportProcessSettings settings,
300318
}
301319
}
302320

303-
return false;
321+
return new ImportReport
322+
{
323+
IsSuccess = false,
324+
Log = ImportLog
325+
};
304326
}
305327

306328
public async Task<List<ChargePoint>> DeDuplicateList(List<ChargePoint> cpList, bool updateDuplicate, CoreReferenceData coreRefData, ImportReport report, bool allowDupeWithDifferentOperator = false, bool fetchExistingFromAPI = false, int dupeDistance = DUPLICATE_DISTANCE_METERS)
@@ -964,6 +986,8 @@ public async Task<ImportReport> PerformImport(ImportProcessSettings settings, AP
964986
ImportReport resultReport = new ImportReport();
965987
resultReport.ProviderDetails = p;
966988
resultReport.IsSuccess = true;
989+
resultReport.IsPreviewMode = settings.IsPreviewMode;
990+
resultReport.PreviewItemLimit = settings.PreviewItemLimit;
967991

968992
try
969993
{
@@ -1035,6 +1059,23 @@ public async Task<ImportReport> PerformImport(ImportProcessSettings settings, AP
10351059
Log("Processing input..");
10361060

10371061
var list = provider.Process(coreRefData);
1062+
resultReport.SourceItemCount = list.Count;
1063+
1064+
if (settings.IsPreviewMode)
1065+
{
1066+
var previewLimit = settings.PreviewItemLimit > 0 ? settings.PreviewItemLimit : 100;
1067+
if (list.Count > previewLimit)
1068+
{
1069+
Log($"Preview mode enabled. Limiting import processing to the first {previewLimit} items out of {list.Count} source items.");
1070+
list = list.Take(previewLimit).ToList();
1071+
}
1072+
else
1073+
{
1074+
Log($"Preview mode enabled. Processing all {list.Count} source items because the source contains fewer than the preview limit of {previewLimit} items.");
1075+
}
1076+
}
1077+
1078+
resultReport.ProcessedSourceItemCount = list.Count;
10381079

10391080
int numAdded = 0;
10401081
int numUpdated = 0;
@@ -1116,6 +1157,9 @@ public async Task<ImportReport> PerformImport(ImportProcessSettings settings, AP
11161157
finalList = finalList.Where(l => l.ID > 0).ToList();
11171158
}
11181159

1160+
numAdded = finalList.Count(p => p.ID == 0);
1161+
numUpdated = finalList.Count(p => p.ID > 0);
1162+
11191163
GC.Collect();
11201164

11211165
if (p.ExportType == ExportType.CSV)
@@ -1134,54 +1178,55 @@ public async Task<ImportReport> PerformImport(ImportProcessSettings settings, AP
11341178

11351179
if (p.ExportType == ExportType.JSONAPI)
11361180
{
1137-
Log("Exporting JSON..");
1138-
//output json
1139-
var fileName = outputPath + "processed_" + p.OutputNamePrefix + ".json";
1140-
p.ExportJSONFile(finalList, fileName);
1141-
1142-
Log("Uploading JSON to API..");
1143-
if (System.IO.File.Exists(fileName))
1181+
if (settings.IsPreviewMode)
11441182
{
1145-
var json = System.IO.File.ReadAllText(fileName);
1146-
await UploadPOIList(json);
1183+
Log("Preview mode enabled. Skipping JSON export upload and import timestamp update.");
11471184
}
1148-
1149-
// notify API of last date of import for each provider
1150-
if (p.DataProviderID > 0)
1185+
else
11511186
{
1152-
await UpdateLastImportDate(p.DataProviderID);
1187+
Log("Exporting JSON..");
1188+
//output json
1189+
var fileName = outputPath + "processed_" + p.OutputNamePrefix + ".json";
1190+
p.ExportJSONFile(finalList, fileName);
1191+
1192+
Log("Uploading JSON to API..");
1193+
if (System.IO.File.Exists(fileName))
1194+
{
1195+
var json = System.IO.File.ReadAllText(fileName);
1196+
await UploadPOIList(json);
1197+
}
1198+
1199+
// notify API of last date of import for each provider
1200+
if (p.DataProviderID > 0)
1201+
{
1202+
await UpdateLastImportDate(p.DataProviderID);
1203+
}
11531204
}
11541205
}
11551206

11561207
if (p.ExportType == ExportType.API && p.IsProductionReady)
11571208
{
1158-
//publish list of locations to OCM via API
1159-
1160-
Log("Publishing via API..");
1161-
foreach (ChargePoint cp in finalList.Where(l => l.AddressInfo.CountryID != null))
1209+
if (settings.IsPreviewMode)
1210+
{
1211+
Log("Preview mode enabled. Skipping API publish and import timestamp update.");
1212+
}
1213+
else
11621214
{
1163-
_client.UpdatePOI(cp, credentials);
1164-
if (cp.ID == 0)
1215+
//publish list of locations to OCM via API
1216+
1217+
Log("Publishing via API..");
1218+
foreach (ChargePoint cp in finalList.Where(l => l.AddressInfo.CountryID != null))
11651219
{
1166-
numAdded++;
1220+
_client.UpdatePOI(cp, credentials);
11671221
}
1168-
else
1222+
1223+
// notify API of last date of import for each provider
1224+
if (p.DataProviderID > 0)
11691225
{
1170-
numUpdated++;
1226+
await UpdateLastImportDate(p.DataProviderID);
11711227
}
11721228
}
11731229

1174-
// notify API of last date of import for each provider
1175-
if (p.DataProviderID > 0)
1176-
{
1177-
await UpdateLastImportDate(p.DataProviderID);
1178-
}
1179-
1180-
}
1181-
else
1182-
{
1183-
numAdded = finalList.Count(p => p.ID == 0);
1184-
numUpdated = finalList.Count(p => p.ID > 0);
11851230
}
11861231

11871232
if (p.ExportType == ExportType.POIModelList)

0 commit comments

Comments
 (0)