Update for new SwissReg XML

lib/oddb2xml/cli.rb

@@ -65,7 +65,12 @@ def run
           threads << download(:firstbase) # https://github.com/zdavatz/oddb2xml/issues/63
         end
         types.each do |type|
-          threads << download(:refdata, type) # refdata
+          begin
+            threads << download(:refdata, type) # refdata
+          rescue error
+            # Should continue even when error #102
+            Oddb2xml.log("Error in downloading refdata #{error}")
+          end
         end
       end
       begin
@@ -311,10 +316,15 @@ def download(what, type = nil)
           end
         end
         @mutex.synchronize do
-          hsh = RefdataExtractor.new(xml, type).to_hash
-          @refdata_types[type] = hsh
-          Oddb2xml.log("RefdataExtractor #{type} added #{hsh.size} keys now #{@refdata_types.keys} items from xml with #{xml.size} bytes")
-          @refdata_types[type]
+          begin
+            hsh = RefdataExtractor.new(xml, type).to_hash
+            @refdata_types[type] = hsh
+            Oddb2xml.log("RefdataExtractor #{type} added #{hsh.size} keys now #{@refdata_types.keys} items from xml with #{xml.size} bytes")
+            @refdata_types[type]
+          rescue error
+            # Should continue even when error https://github.com/zdavatz/oddb2xml/issues/102
+            Oddb2xml.log("Error in RefdataExtractor #{error}")
+          end
         end
 
       when :firstbase

lib/oddb2xml/downloader.rb

@@ -241,9 +241,9 @@ def download
   end
 
   class RefdataDownloader < Downloader
+    include DownloadMethod
     def initialize(options = {}, type = :pharma)
-      @type = (type == :pharma ? "Pharma" : "NonPharma")
-      url = "https://refdatabase.refdata.ch/Service/Article.asmx?WSDL"
+      url = "https://files.refdata.ch/simis-public-prod/Articles/1.0/Refdata.Articles.zip"
       super(options, url)
     end
 
@@ -258,46 +258,10 @@ def init
     end
 
     def download
-      begin
-        @file2save = File.join(DOWNLOADS, "refdata_#{@type}.xml")
-        soap = %(<?xml version="1.0" encoding="UTF-8"?>
-<SOAP-ENV:Envelope xmlns:SOAP-ENV="http://schemas.xmlsoap.org/soap/envelope/" xmlns:ns1="http://refdatabase.refdata.ch/Article_in" xmlns:ns2="http://refdatabase.refdata.ch/">
-  <SOAP-ENV:Body>
-    <ns2:DownloadArticleInput>
-      <ns1:ATYPE>#{@type.upcase}</ns1:ATYPE>
-    </ns2:DownloadArticleInput>
-  </SOAP-ENV:Body>
-  </SOAP-ENV:Envelope>
-</ns1:ATYPE></ns2:DownloadArticleInput></SOAP-ENV:Body>
-)
-        report_download(@url, @file2save)
-        return IO.read(@file2save) if Oddb2xml.skip_download? && File.exist?(@file2save)
-        FileUtils.rm_f(@file2save, verbose: true)
-        response = @client.call(:download, xml: soap)
-        if response.success?
-          if (xml = response.to_xml)
-            xml = File.read(File.join(Oddb2xml::SpecData, File.basename(@file2save))) if defined?(RSpec)
-            response = nil # win
-            FileUtils.makedirs(DOWNLOADS)
-            File.open(@file2save, "w+") { |file| file.write xml }
-            if @options[:artikelstamm]
-              cmd = "xmllint --format --output #{@file2save} #{@file2save}"
-              Oddb2xml.log(cmd)
-              system(cmd)
-            end
-          else
-            # received broken data or internal error
-            raise StandardError
-          end
-        else
-          raise Timeout::Error
-        end
-      rescue HTTPI::SSLError
-        exit # catch me in Cli class
-      rescue Timeout::Error, Errno::ETIMEDOUT
-        retrievable? ? retry : raise
-      end
-      xml
+      filename = "Refdata.Articles.zip"
+      download_as(filename, "w+")
+      content = read_xml_from_zip(/Refdata.Articles.xml/, File.join(DOWNLOADS, filename))
+      content
     end
   end
 

lib/oddb2xml/extractor.rb

@@ -208,10 +208,13 @@ def initialize(xml, type)
 
     def to_hash
       data = {}
-      result = SwissRegArticleEntry.parse(@xml.sub(STRIP_FOR_SAX_MACHINE, ""), lazy: true)
-      items = result.ARTICLE.ITEM
-      items.each do |pac|
-        ean13 = (gtin = pac.GTIN.to_s) ? gtin : "0"
+      result = SwissRegArticles.parse(@xml.sub(STRIP_FOR_SAX_MACHINE, ""), lazy: true)
+      result.Article.each do |article|
+        article_type = article.MedicinalProduct.ProductClassification.ProductClass
+        if article_type != @type
+          next
+        end
+        ean13 = @type == "PHARMA" ? article.PackagedProduct.DataCarrierIdentifier : article.MedicinalProduct.Identifier
         if ean13.size < 13
           puts "Refdata #{@type} use 13 chars not #{ean13.size} for #{ean13}" if $VERBOSE
           ean13 = ean13.rjust(13, "0")
@@ -220,21 +223,29 @@ def to_hash
           puts "Refdata #{@type} remove leading '0' for #{ean13}" if $VERBOSE
           ean13 = ean13[1..-1]
         end
-        # but in refdata_nonPharma we have a about 700 GTINs which are 14 characters and longer
         item = {}
         item[:ean13] = ean13
-        item[:no8] = pac.SWMC_AUTHNR
+        item[:no8] = article.PackagedProduct.RegulatedAuthorisationIdentifier || ""
         item[:data_origin] = "refdata"
         item[:refdata] = true
-        item[:_type] = (typ = pac.ATYPE.downcase.to_sym) ? typ : ""
-        item[:last_change] = (date = Time.parse(pac.DT).to_s) ? date : "" # Date and time of last data change
-        item[:desc_de] = (dscr = pac.NAME_DE) ? dscr : ""
-        item[:desc_fr] = (dscr = pac.NAME_FR) ? dscr : ""
-        item[:desc_it] = item[:desc_de] # refdata has no italian name
-        item[:atc_code] = (code = pac.ATC) ? code.to_s : ""
-        item[:company_name] = (nam = pac.AUTH_HOLDER_NAME) ? nam : ""
-        item[:company_ean] = (gln = pac.AUTH_HOLDER_GLN) ? gln : ""
-        data[item[:ean13]] = item
+        item[:_type] = @type.downcase.to_sym
+        item[:last_change] = "" # TODO: Date and time of last data change
+        item[:desc_de] = ""
+        item[:desc_fr] = ""
+        item[:desc_it] = ""
+        article.PackagedProduct.Name.each do |name|
+          if name.Language == "DE"
+            item[:desc_de] = name.FullName
+          elsif name.Language == "FR"
+            item[:desc_fr] = name.FullName
+          elsif name.Language == "IT"
+            item[:desc_it] = name.FullName
+          end
+        end
+        item[:atc_code] = article.MedicinalProduct.ProductClassification.Atc || ""
+        item[:company_name] = article.PackagedProduct.Holder.Name || ""
+        item[:company_ean] = article.PackagedProduct.Holder.Identifier || ""
+        data[ean13] = item
       end
       data
     end

lib/oddb2xml/xml_definitions.rb

@@ -238,29 +238,69 @@ class MedicalInformationsEntry
   element :medicalInformations, class: MedicalInformationsContent
 end
 
-class SwissRegItemContentContent
+class SwissRegProductClassification
   include SAXMachine
-  attribute :DT
-  element :ATYPE
-  element :GTIN
-  element :PHAR
-  element :SWMC_AUTHNR
-  element :NAME_DE
-  element :NAME_FR
-  element :ADDSCR
-  element :ATC
-  element :AUTH_HOLDER_NAME
-  element :AUTH_HOLDER_GLN
+  element :ProductClass
+  element :Atc
 end
 
-class SwissRegArticleContent
+class SwissRegMedicinalProduct
   include SAXMachine
-  attribute :CREATION_DATETIME
-  elements :ITEM, class: SwissRegItemContentContent
+  element :Identifier
+  element :Domain
+  element :LegalStatusOfSupply
+  element :RegulatedAuthorisationIdentifier
+  element :ProductClassification, class: SwissRegProductClassification
 end
 
-class SwissRegArticleEntry
+class SwissRegHolder
   include SAXMachine
-  element :CREATION_DATETIME
-  element :ARTICLE, class: SwissRegArticleContent
+  element :Identifier
+  element :Name
+end
+
+class SwissRegName
+  include SAXMachine
+  element :Language
+  element :FullName
+end
+
+class SwissRegMarketingStatus
+  include SAXMachine
+  element :DateStart
+end
+
+class SwissRegPackagedProduct
+  include SAXMachine
+  element :Identifier
+  element :RegulatedAuthorisationIdentifier
+  element :DataCarrierIdentifier
+  element :Holder, class: SwissRegHolder
+  elements :Name, class: SwissRegName
+  element :MarketingStatus, class: SwissRegMarketingStatus
+end
+
+class SwissRegArticle
+  include SAXMachine
+  element :MedicinalProduct, class: SwissRegMedicinalProduct
+  element :PackagedProduct, class: SwissRegPackagedProduct
+end
+
+class SwissRegDocumentReference
+  include SAXMachine
+  element :Language
+  element :Url
+end
+
+class SwissRegProductPrice
+  include SAXMachine
+  element :RetailPrice
+end
+
+class SwissRegArticles
+  include SAXMachine
+  elements :Article, class: SwissRegArticle
+  elements :DocumentReference, class: SwissRegDocumentReference
+  element :Hpc
+  element :ProductPrice, class: SwissRegProductPrice
 end