Coverage for /var/srv/projects/api.amasfac.comuna18.com/tmp/venv/lib/python3.9/site-packages/import_export/resources.py: 33%
564 statements
« prev ^ index » next coverage.py v6.4.4, created at 2023-07-17 14:22 -0600
1import functools
2import logging
3import traceback
4import warnings
5from collections import OrderedDict
6from copy import deepcopy
8import tablib
9from diff_match_patch import diff_match_patch
10from django.conf import settings
11from django.core.exceptions import (
12 FieldDoesNotExist,
13 ImproperlyConfigured,
14 ValidationError,
15)
16from django.core.management.color import no_style
17from django.core.paginator import Paginator
18from django.db import connections, router
19from django.db.models.fields.related import ForeignObjectRel
20from django.db.models.query import QuerySet
21from django.db.transaction import TransactionManagementError, set_rollback
22from django.utils.encoding import force_str
23from django.utils.safestring import mark_safe
25from . import widgets
26from .fields import Field
27from .instance_loaders import ModelInstanceLoader
28from .results import Error, Result, RowResult
29from .utils import atomic_if_using_transaction
31logger = logging.getLogger(__name__)
32# Set default logging handler to avoid "No handler found" warnings.
33logger.addHandler(logging.NullHandler())
36def get_related_model(field):
37 if hasattr(field, 'related_model'): 37 ↛ exitline 37 didn't return from function 'get_related_model', because the condition on line 37 was never false
38 return field.related_model
40def has_natural_foreign_key(model):
41 """
42 Determine if a model has natural foreign key functions
43 """
44 return (hasattr(model, "natural_key") and hasattr(model.objects, "get_by_natural_key"))
46class ResourceOptions:
47 """
48 The inner Meta class allows for class-level configuration of how the
49 Resource should behave. The following options are available:
50 """
52 model = None
53 """
54 Django Model class. It is used to introspect available
55 fields.
57 """
58 fields = None
59 """
60 Controls what introspected fields the Resource should include. A whitelist
61 of fields.
62 """
64 exclude = None
65 """
66 Controls what introspected fields the Resource should
67 NOT include. A blacklist of fields.
68 """
70 instance_loader_class = None
71 """
72 Controls which class instance will take
73 care of loading existing objects.
74 """
76 import_id_fields = ['id']
77 """
78 Controls which object fields will be used to
79 identify existing instances.
80 """
82 export_order = None
83 """
84 Controls export order for columns.
85 """
87 widgets = None
88 """
89 This dictionary defines widget kwargs for fields.
90 """
92 use_transactions = None
93 """
94 Controls if import should use database transactions. Default value is
95 ``None`` meaning ``settings.IMPORT_EXPORT_USE_TRANSACTIONS`` will be
96 evaluated.
97 """
99 skip_unchanged = False
100 """
101 Controls if the import should skip unchanged records. Default value is
102 False
103 """
105 report_skipped = True
106 """
107 Controls if the result reports skipped rows. Default value is True
108 """
110 clean_model_instances = False
111 """
112 Controls whether ``instance.full_clean()`` is called during the import
113 process to identify potential validation errors for each (non skipped) row.
114 The default value is False.
115 """
117 chunk_size = None
118 """
119 Controls the chunk_size argument of Queryset.iterator or,
120 if prefetch_related is used, the per_page attribute of Paginator.
121 """
123 skip_diff = False
124 """
125 Controls whether or not an instance should be diffed following import.
126 By default, an instance is copied prior to insert, update or delete.
127 After each row is processed, the instance's copy is diffed against the original, and the value
128 stored in each ``RowResult``.
129 If diffing is not required, then disabling the diff operation by setting this value to ``True``
130 improves performance, because the copy and comparison operations are skipped for each row.
131 If enabled, then ``skip_row()`` checks do not execute, because 'skip' logic requires
132 comparison between the stored and imported versions of a row.
133 If enabled, then HTML row reports are also not generated (see ``skip_html_diff``).
134 The default value is False.
135 """
137 skip_html_diff = False
138 """
139 Controls whether or not a HTML report is generated after each row.
140 By default, the difference between a stored copy and an imported instance
141 is generated in HTML form and stored in each ``RowResult``.
142 The HTML report is used to present changes on the confirmation screen in the admin site,
143 hence when this value is ``True``, then changes will not be presented on the confirmation
144 screen.
145 If the HTML report is not required, then setting this value to ``True`` improves performance,
146 because the HTML generation is skipped for each row.
147 This is a useful optimization when importing large datasets.
148 The default value is False.
149 """
151 use_bulk = False
152 """
153 Controls whether import operations should be performed in bulk.
154 By default, an object's save() method is called for each row in a data set.
155 When bulk is enabled, objects are saved using bulk operations.
156 """
158 batch_size = 1000
159 """
160 The batch_size parameter controls how many objects are created in a single query.
161 The default is to create objects in batches of 1000.
162 See `bulk_create() <https://docs.djangoproject.com/en/dev/ref/models/querysets/#bulk-create>`_.
163 This parameter is only used if ``use_bulk`` is True.
164 """
166 force_init_instance = False
167 """
168 If True, this parameter will prevent imports from checking the database for existing instances.
169 Enabling this parameter is a performance enhancement if your import dataset is guaranteed to
170 contain new instances.
171 """
173 using_db = None
174 """
175 DB Connection name to use for db transactions. If not provided,
176 ``router.db_for_write(model)`` will be evaluated and if it's missing,
177 DEFAULT_DB_ALIAS constant ("default") is used.
178 """
180 store_row_values = False
181 """
182 If True, each row's raw data will be stored in each row result.
183 Enabling this parameter will increase the memory usage during import
184 which should be considered when importing large datasets.
185 """
187 use_natural_foreign_keys = False
188 """
189 If True, use_natural_foreign_keys = True will be passed to all foreign
190 key widget fields whose models support natural foreign keys. That is,
191 the model has a natural_key function and the manager has a
192 get_by_natural_key function.
193 """
196class DeclarativeMetaclass(type):
198 def __new__(cls, name, bases, attrs):
199 declared_fields = []
200 meta = ResourceOptions()
202 # If this class is subclassing another Resource, add that Resource's
203 # fields. Note that we loop over the bases in *reverse*. This is
204 # necessary in order to preserve the correct order of fields.
205 for base in bases[::-1]:
206 if hasattr(base, 'fields'): 206 ↛ 205line 206 didn't jump to line 205, because the condition on line 206 was never false
207 declared_fields = list(base.fields.items()) + declared_fields
208 # Collect the Meta options
209 options = getattr(base, 'Meta', None)
210 for option in [option for option in dir(options) 210 ↛ 212line 210 didn't jump to line 212, because the loop on line 210 never started
211 if not option.startswith('_') and hasattr(options, option)]:
212 setattr(meta, option, getattr(options, option))
214 # Add direct fields
215 for field_name, obj in attrs.copy().items():
216 if isinstance(obj, Field):
217 field = attrs.pop(field_name)
218 if not field.column_name: 218 ↛ 220line 218 didn't jump to line 220, because the condition on line 218 was never false
219 field.column_name = field_name
220 declared_fields.append((field_name, field))
222 attrs['fields'] = OrderedDict(declared_fields)
223 new_class = super().__new__(cls, name, bases, attrs)
225 # Add direct options
226 options = getattr(new_class, 'Meta', None)
227 for option in [option for option in dir(options)
228 if not option.startswith('_') and hasattr(options, option)]:
229 setattr(meta, option, getattr(options, option))
230 new_class._meta = meta
232 return new_class
235class Diff:
236 def __init__(self, resource, instance, new):
237 self.left = self._export_resource_fields(resource, instance)
238 self.right = []
239 self.new = new
241 def compare_with(self, resource, instance, dry_run=False):
242 self.right = self._export_resource_fields(resource, instance)
244 def as_html(self):
245 data = []
246 dmp = diff_match_patch()
247 for v1, v2 in zip(self.left, self.right):
248 if v1 != v2 and self.new:
249 v1 = ""
250 diff = dmp.diff_main(force_str(v1), force_str(v2))
251 dmp.diff_cleanupSemantic(diff)
252 html = dmp.diff_prettyHtml(diff)
253 html = mark_safe(html)
254 data.append(html)
255 return data
257 def _export_resource_fields(self, resource, instance):
258 return [resource.export_field(f, instance) if instance else "" for f in resource.get_user_visible_fields()]
261class Resource(metaclass=DeclarativeMetaclass):
262 """
263 Resource defines how objects are mapped to their import and export
264 representations and handle importing and exporting data.
265 """
267 def __init__(self, **kwargs):
268 """
269 kwargs:
270 An optional dict of kwargs.
271 Subclasses can use kwargs to pass dynamic values to enhance import / exports.
272 """
273 # The fields class attribute is the *class-wide* definition of
274 # fields. Because a particular *instance* of the class might want to
275 # alter self.fields, we create self.fields here by copying cls.fields.
276 # Instances should always modify self.fields; they should not modify
277 # cls.fields.
278 self.fields = deepcopy(self.fields)
280 # lists to hold model instances in memory when bulk operations are enabled
281 self.create_instances = list()
282 self.update_instances = list()
283 self.delete_instances = list()
285 @classmethod
286 def get_result_class(self):
287 """
288 Returns the class used to store the result of an import.
289 """
290 return Result
292 @classmethod
293 def get_row_result_class(self):
294 """
295 Returns the class used to store the result of a row import.
296 """
297 return RowResult
299 @classmethod
300 def get_error_result_class(self):
301 """
302 Returns the class used to store an error resulting from an import.
303 """
304 return Error
306 @classmethod
307 def get_diff_class(self):
308 """
309 Returns the class used to display the diff for an imported instance.
310 """
311 return Diff
313 @classmethod
314 def get_db_connection_name(self):
315 if self._meta.using_db is None:
316 return router.db_for_write(self._meta.model)
317 else:
318 return self._meta.using_db
320 def get_use_transactions(self):
321 if self._meta.use_transactions is None:
322 return getattr(settings, 'IMPORT_EXPORT_USE_TRANSACTIONS', True)
323 else:
324 return self._meta.use_transactions
326 def get_chunk_size(self):
327 if self._meta.chunk_size is None:
328 return getattr(settings, 'IMPORT_EXPORT_CHUNK_SIZE', 100)
329 else:
330 return self._meta.chunk_size
332 def get_fields(self, **kwargs):
333 """
334 Returns fields sorted according to
335 :attr:`~import_export.resources.ResourceOptions.export_order`.
336 """
337 return [self.fields[f] for f in self.get_export_order()]
339 def get_field_name(self, field):
340 """
341 Returns the field name for a given field.
342 """
343 for field_name, f in self.fields.items():
344 if f == field:
345 return field_name
346 raise AttributeError("Field %s does not exists in %s resource" % (
347 field, self.__class__))
349 def init_instance(self, row=None):
350 """
351 Initializes an object. Implemented in
352 :meth:`import_export.resources.ModelResource.init_instance`.
353 """
354 raise NotImplementedError()
356 def get_instance(self, instance_loader, row):
357 """
358 If all 'import_id_fields' are present in the dataset, calls
359 the :doc:`InstanceLoader <api_instance_loaders>`. Otherwise,
360 returns `None`.
361 """
362 import_id_fields = [
363 self.fields[f] for f in self.get_import_id_fields()
364 ]
365 for field in import_id_fields:
366 if field.column_name not in row:
367 return
368 return instance_loader.get_instance(row)
370 def get_or_init_instance(self, instance_loader, row):
371 """
372 Either fetches an already existing instance or initializes a new one.
373 """
374 if not self._meta.force_init_instance:
375 instance = self.get_instance(instance_loader, row)
376 if instance:
377 return (instance, False)
378 return (self.init_instance(row), True)
380 def get_import_id_fields(self):
381 """
382 """
383 return self._meta.import_id_fields
385 def get_bulk_update_fields(self):
386 """
387 Returns the fields to be included in calls to bulk_update().
388 ``import_id_fields`` are removed because `id` fields cannot be supplied to bulk_update().
389 """
390 return [f for f in self.fields if f not in self._meta.import_id_fields]
392 def bulk_create(self, using_transactions, dry_run, raise_errors, batch_size=None, result=None):
393 """
394 Creates objects by calling ``bulk_create``.
395 """
396 try:
397 if len(self.create_instances) > 0:
398 if not using_transactions and dry_run:
399 pass
400 else:
401 self._meta.model.objects.bulk_create(self.create_instances, batch_size=batch_size)
402 except Exception as e:
403 self.handle_import_error(result, e, raise_errors)
404 finally:
405 self.create_instances.clear()
407 def bulk_update(self, using_transactions, dry_run, raise_errors, batch_size=None, result=None):
408 """
409 Updates objects by calling ``bulk_update``.
410 """
411 try:
412 if len(self.update_instances) > 0:
413 if not using_transactions and dry_run:
414 pass
415 else:
416 self._meta.model.objects.bulk_update(self.update_instances, self.get_bulk_update_fields(),
417 batch_size=batch_size)
418 except Exception as e:
419 self.handle_import_error(result, e, raise_errors)
420 finally:
421 self.update_instances.clear()
423 def bulk_delete(self, using_transactions, dry_run, raise_errors, result=None):
424 """
425 Deletes objects by filtering on a list of instances to be deleted,
426 then calling ``delete()`` on the entire queryset.
427 """
428 try:
429 if len(self.delete_instances) > 0:
430 if not using_transactions and dry_run:
431 pass
432 else:
433 delete_ids = [o.pk for o in self.delete_instances]
434 self._meta.model.objects.filter(pk__in=delete_ids).delete()
435 except Exception as e:
436 self.handle_import_error(result, e, raise_errors)
437 finally:
438 self.delete_instances.clear()
440 def validate_instance(self, instance, import_validation_errors=None, validate_unique=True):
441 """
442 Takes any validation errors that were raised by
443 :meth:`~import_export.resources.Resource.import_obj`, and combines them
444 with validation errors raised by the instance's ``full_clean()``
445 method. The combined errors are then re-raised as single, multi-field
446 ValidationError.
448 If the ``clean_model_instances`` option is False, the instances's
449 ``full_clean()`` method is not called, and only the errors raised by
450 ``import_obj()`` are re-raised.
451 """
452 if import_validation_errors is None:
453 errors = {}
454 else:
455 errors = import_validation_errors.copy()
456 if self._meta.clean_model_instances:
457 try:
458 instance.full_clean(
459 exclude=errors.keys(),
460 validate_unique=validate_unique,
461 )
462 except ValidationError as e:
463 errors = e.update_error_dict(errors)
465 if errors:
466 raise ValidationError(errors)
468 def save_instance(self, instance, is_create, using_transactions=True, dry_run=False):
469 """
470 Takes care of saving the object to the database.
472 Objects can be created in bulk if ``use_bulk`` is enabled.
474 :param instance: The instance of the object to be persisted.
475 :param is_create: A boolean flag to indicate whether this is a new object
476 to be created, or an existing object to be updated.
477 :param using_transactions: A flag to indicate whether db transactions are used.
478 :param dry_run: A flag to indicate dry-run mode.
479 """
480 self.before_save_instance(instance, using_transactions, dry_run)
481 if self._meta.use_bulk:
482 if is_create:
483 self.create_instances.append(instance)
484 else:
485 self.update_instances.append(instance)
486 else:
487 if not using_transactions and dry_run:
488 # we don't have transactions and we want to do a dry_run
489 pass
490 else:
491 instance.save()
492 self.after_save_instance(instance, using_transactions, dry_run)
494 def before_save_instance(self, instance, using_transactions, dry_run):
495 """
496 Override to add additional logic. Does nothing by default.
497 """
498 pass
500 def after_save_instance(self, instance, using_transactions, dry_run):
501 """
502 Override to add additional logic. Does nothing by default.
503 """
504 pass
506 def delete_instance(self, instance, using_transactions=True, dry_run=False):
507 """
508 Calls :meth:`instance.delete` as long as ``dry_run`` is not set.
509 If ``use_bulk`` then instances are appended to a list for bulk import.
510 """
511 self.before_delete_instance(instance, dry_run)
512 if self._meta.use_bulk:
513 self.delete_instances.append(instance)
514 else:
515 if not using_transactions and dry_run:
516 # we don't have transactions and we want to do a dry_run
517 pass
518 else:
519 instance.delete()
520 self.after_delete_instance(instance, dry_run)
522 def before_delete_instance(self, instance, dry_run):
523 """
524 Override to add additional logic. Does nothing by default.
525 """
526 pass
528 def after_delete_instance(self, instance, dry_run):
529 """
530 Override to add additional logic. Does nothing by default.
531 """
532 pass
534 def import_field(self, field, obj, data, is_m2m=False, **kwargs):
535 """
536 Calls :meth:`import_export.fields.Field.save` if ``Field.attribute``
537 is specified, and ``Field.column_name`` is found in ``data``.
538 """
539 if field.attribute and field.column_name in data:
540 field.save(obj, data, is_m2m, **kwargs)
542 def get_import_fields(self):
543 return self.get_fields()
545 def import_obj(self, obj, data, dry_run, **kwargs):
546 """
547 Traverses every field in this Resource and calls
548 :meth:`~import_export.resources.Resource.import_field`. If
549 ``import_field()`` results in a ``ValueError`` being raised for
550 one of more fields, those errors are captured and reraised as a single,
551 multi-field ValidationError."""
552 errors = {}
553 for field in self.get_import_fields():
554 if isinstance(field.widget, widgets.ManyToManyWidget):
555 continue
556 try:
557 self.import_field(field, obj, data, **kwargs)
558 except ValueError as e:
559 errors[field.attribute] = ValidationError(
560 force_str(e), code="invalid")
561 if errors:
562 raise ValidationError(errors)
564 def save_m2m(self, obj, data, using_transactions, dry_run):
565 """
566 Saves m2m fields.
568 Model instance need to have a primary key value before
569 a many-to-many relationship can be used.
570 """
571 if (not using_transactions and dry_run) or self._meta.use_bulk:
572 # we don't have transactions and we want to do a dry_run
573 # OR use_bulk is enabled (m2m operations are not supported for bulk operations)
574 pass
575 else:
576 for field in self.get_import_fields():
577 if not isinstance(field.widget, widgets.ManyToManyWidget):
578 continue
579 self.import_field(field, obj, data, True)
581 def for_delete(self, row, instance):
582 """
583 Returns ``True`` if ``row`` importing should delete instance.
585 Default implementation returns ``False``.
586 Override this method to handle deletion.
587 """
588 return False
590 def skip_row(self, instance, original, row, import_validation_errors=None):
591 """
592 Returns ``True`` if ``row`` importing should be skipped.
594 Default implementation returns ``False`` unless skip_unchanged == True
595 and skip_diff == False.
597 If skip_diff is True, then no comparisons can be made because ``original``
598 will be None.
600 When left unspecified, skip_diff and skip_unchanged both default to ``False``,
601 and rows are never skipped.
603 By default, rows are not skipped if validation errors have been detected
604 during import. You can change this behavior and choose to ignore validation
605 errors by overriding this method.
607 Override this method to handle skipping rows meeting certain
608 conditions.
610 Use ``super`` if you want to preserve default handling while overriding
611 ::
612 class YourResource(ModelResource):
613 def skip_row(self, instance, original, row, import_validation_errors=None):
614 # Add code here
615 return super().skip_row(instance, original, row, import_validation_errors=import_validation_errors)
617 """
618 if not self._meta.skip_unchanged or self._meta.skip_diff or import_validation_errors:
619 return False
620 for field in self.get_import_fields():
621 # For fields that are models.fields.related.ManyRelatedManager
622 # we need to compare the results
623 if isinstance(field.widget, widgets.ManyToManyWidget):
624 # #1437 - handle m2m field not present in import file
625 if field.column_name not in row.keys():
626 continue
627 # m2m instance values are taken from the 'row' because they
628 # have not been written to the 'instance' at this point
629 instance_values = list(field.clean(row))
630 original_values = list(field.get_value(original).all())
631 if len(instance_values) != len(original_values):
632 return False
634 if sorted(v.pk for v in instance_values) != sorted(v.pk for v in original_values):
635 return False
636 else:
637 if field.get_value(instance) != field.get_value(original):
638 return False
639 return True
641 def get_diff_headers(self):
642 """
643 Diff representation headers.
644 """
645 return self.get_user_visible_headers()
647 def before_import(self, dataset, using_transactions, dry_run, **kwargs):
648 """
649 Override to add additional logic. Does nothing by default.
650 """
651 pass
653 def after_import(self, dataset, result, using_transactions, dry_run, **kwargs):
654 """
655 Override to add additional logic. Does nothing by default.
656 """
657 pass
659 def before_import_row(self, row, row_number=None, **kwargs):
660 """
661 Override to add additional logic. Does nothing by default.
662 """
663 pass
665 def after_import_row(self, row, row_result, row_number=None, **kwargs):
666 """
667 Override to add additional logic. Does nothing by default.
668 """
669 pass
671 def after_import_instance(self, instance, new, row_number=None, **kwargs):
672 """
673 Override to add additional logic. Does nothing by default.
674 """
675 pass
677 def handle_import_error(self, result, error, raise_errors=False):
678 logger.debug(error, exc_info=error)
679 if result:
680 tb_info = traceback.format_exc()
681 result.append_base_error(self.get_error_result_class()(error, tb_info))
682 if raise_errors:
683 raise
685 def import_row(self, row, instance_loader, using_transactions=True, dry_run=False, raise_errors=None, **kwargs):
686 """
687 Imports data from ``tablib.Dataset``. Refer to :doc:`import_workflow`
688 for a more complete description of the whole import process.
690 :param row: A ``dict`` of the row to import
692 :param instance_loader: The instance loader to be used to load the row
694 :param using_transactions: If ``using_transactions`` is set, a transaction
695 is being used to wrap the import
697 :param dry_run: If ``dry_run`` is set, or error occurs, transaction
698 will be rolled back.
699 """
700 if raise_errors is not None:
701 warnings.warn(
702 "raise_errors argument is deprecated and will be removed in a future release.",
703 category=DeprecationWarning
704 )
706 skip_diff = self._meta.skip_diff
707 row_result = self.get_row_result_class()()
708 if self._meta.store_row_values:
709 row_result.row_values = row
710 original = None
711 try:
712 self.before_import_row(row, **kwargs)
713 instance, new = self.get_or_init_instance(instance_loader, row)
714 self.after_import_instance(instance, new, **kwargs)
715 if new:
716 row_result.import_type = RowResult.IMPORT_TYPE_NEW
717 else:
718 row_result.import_type = RowResult.IMPORT_TYPE_UPDATE
719 row_result.new_record = new
720 if not skip_diff:
721 original = deepcopy(instance)
722 diff = self.get_diff_class()(self, original, new)
723 if self.for_delete(row, instance):
724 if new:
725 row_result.import_type = RowResult.IMPORT_TYPE_SKIP
726 if not skip_diff:
727 diff.compare_with(self, None, dry_run)
728 else:
729 row_result.import_type = RowResult.IMPORT_TYPE_DELETE
730 row_result.add_instance_info(instance)
731 self.delete_instance(instance, using_transactions, dry_run)
732 if not skip_diff:
733 diff.compare_with(self, None, dry_run)
734 else:
735 import_validation_errors = {}
736 try:
737 self.import_obj(instance, row, dry_run, **kwargs)
738 except ValidationError as e:
739 # Validation errors from import_obj() are passed on to
740 # validate_instance(), where they can be combined with model
741 # instance validation errors if necessary
742 import_validation_errors = e.update_error_dict(import_validation_errors)
744 if self.skip_row(instance, original, row, import_validation_errors):
745 row_result.import_type = RowResult.IMPORT_TYPE_SKIP
746 else:
747 self.validate_instance(instance, import_validation_errors)
748 self.save_instance(instance, new, using_transactions, dry_run)
749 self.save_m2m(instance, row, using_transactions, dry_run)
750 row_result.add_instance_info(instance)
751 if not skip_diff:
752 diff.compare_with(self, instance, dry_run)
754 if not skip_diff and not self._meta.skip_html_diff:
755 row_result.diff = diff.as_html()
756 self.after_import_row(row, row_result, **kwargs)
758 except ValidationError as e:
759 row_result.import_type = RowResult.IMPORT_TYPE_INVALID
760 row_result.validation_error = e
761 except Exception as e:
762 row_result.import_type = RowResult.IMPORT_TYPE_ERROR
763 # There is no point logging a transaction error for each row
764 # when only the original error is likely to be relevant
765 if not isinstance(e, TransactionManagementError):
766 logger.debug(e, exc_info=e)
767 tb_info = traceback.format_exc()
768 row_result.errors.append(self.get_error_result_class()(e, tb_info, row))
770 return row_result
772 def import_data(self, dataset, dry_run=False, raise_errors=False,
773 use_transactions=None, collect_failed_rows=False,
774 rollback_on_validation_errors=False, **kwargs):
775 """
776 Imports data from ``tablib.Dataset``. Refer to :doc:`import_workflow`
777 for a more complete description of the whole import process.
779 :param dataset: A ``tablib.Dataset``
781 :param raise_errors: Whether errors should be printed to the end user
782 or raised regularly.
784 :param use_transactions: If ``True`` the import process will be processed
785 inside a transaction.
787 :param collect_failed_rows: If ``True`` the import process will collect
788 failed rows.
790 :param rollback_on_validation_errors: If both ``use_transactions`` and ``rollback_on_validation_errors``
791 are set to ``True``, the import process will be rolled back in case of ValidationError.
793 :param dry_run: If ``dry_run`` is set, or an error occurs, if a transaction
794 is being used, it will be rolled back.
795 """
797 if use_transactions is None:
798 use_transactions = self.get_use_transactions()
800 db_connection = self.get_db_connection_name()
801 connection = connections[db_connection]
802 supports_transactions = getattr(connection.features, "supports_transactions", False)
804 if use_transactions and not supports_transactions:
805 raise ImproperlyConfigured
807 using_transactions = (use_transactions or dry_run) and supports_transactions
809 if self._meta.batch_size is not None and (not isinstance(self._meta.batch_size, int) or self._meta.batch_size < 0):
810 raise ValueError("Batch size must be a positive integer")
812 with atomic_if_using_transaction(using_transactions, using=db_connection):
813 result = self.import_data_inner(
814 dataset, dry_run, raise_errors, using_transactions, collect_failed_rows, **kwargs)
815 if using_transactions and (dry_run or result.has_errors() or
816 (rollback_on_validation_errors and result.has_validation_errors())):
817 set_rollback(True, using=db_connection)
818 return result
820 def import_data_inner(
821 self, dataset, dry_run, raise_errors, using_transactions,
822 collect_failed_rows, rollback_on_validation_errors=None, **kwargs):
824 if rollback_on_validation_errors is not None:
825 warnings.warn(
826 "rollback_on_validation_errors argument is deprecated and will be removed in a future release.",
827 category=DeprecationWarning
828 )
830 result = self.get_result_class()()
831 result.diff_headers = self.get_diff_headers()
832 result.total_rows = len(dataset)
833 db_connection = self.get_db_connection_name()
835 try:
836 with atomic_if_using_transaction(using_transactions, using=db_connection):
837 self.before_import(dataset, using_transactions, dry_run, **kwargs)
838 except Exception as e:
839 self.handle_import_error(result, e, raise_errors)
841 instance_loader = self._meta.instance_loader_class(self, dataset)
843 # Update the total in case the dataset was altered by before_import()
844 result.total_rows = len(dataset)
846 if collect_failed_rows:
847 result.add_dataset_headers(dataset.headers)
849 for i, data_row in enumerate(dataset, 1):
850 row = OrderedDict(zip(dataset.headers, data_row))
851 with atomic_if_using_transaction(using_transactions and not self._meta.use_bulk, using=db_connection):
852 row_result = self.import_row(
853 row,
854 instance_loader,
855 using_transactions=using_transactions,
856 dry_run=dry_run,
857 row_number=i,
858 **kwargs
859 )
860 if self._meta.use_bulk:
861 # persist a batch of rows
862 # because this is a batch, any exceptions are logged and not associated
863 # with a specific row
864 if len(self.create_instances) == self._meta.batch_size:
865 with atomic_if_using_transaction(using_transactions, using=db_connection):
866 self.bulk_create(using_transactions, dry_run, raise_errors,
867 batch_size=self._meta.batch_size, result=result)
868 if len(self.update_instances) == self._meta.batch_size:
869 with atomic_if_using_transaction(using_transactions, using=db_connection):
870 self.bulk_update(using_transactions, dry_run, raise_errors,
871 batch_size=self._meta.batch_size, result=result)
872 if len(self.delete_instances) == self._meta.batch_size:
873 with atomic_if_using_transaction(using_transactions, using=db_connection):
874 self.bulk_delete(using_transactions, dry_run, raise_errors, result=result)
876 result.increment_row_result_total(row_result)
878 if row_result.errors:
879 if collect_failed_rows:
880 result.append_failed_row(row, row_result.errors[0])
881 if raise_errors:
882 raise row_result.errors[-1].error
883 elif row_result.validation_error:
884 result.append_invalid_row(i, row, row_result.validation_error)
885 if collect_failed_rows:
886 result.append_failed_row(row, row_result.validation_error)
887 if raise_errors:
888 raise row_result.validation_error
889 if (row_result.import_type != RowResult.IMPORT_TYPE_SKIP or
890 self._meta.report_skipped):
891 result.append_row_result(row_result)
893 if self._meta.use_bulk:
894 # bulk persist any instances which are still pending
895 with atomic_if_using_transaction(using_transactions, using=db_connection):
896 self.bulk_create(using_transactions, dry_run, raise_errors, result=result)
897 self.bulk_update(using_transactions, dry_run, raise_errors, result=result)
898 self.bulk_delete(using_transactions, dry_run, raise_errors, result=result)
900 try:
901 with atomic_if_using_transaction(using_transactions, using=db_connection):
902 self.after_import(dataset, result, using_transactions, dry_run, **kwargs)
903 except Exception as e:
904 self.handle_import_error(result, e, raise_errors)
906 return result
908 def get_export_order(self):
909 order = tuple(self._meta.export_order or ())
910 return order + tuple(k for k in self.fields if k not in order)
912 def before_export(self, queryset, *args, **kwargs):
913 """
914 Override to add additional logic. Does nothing by default.
915 """
916 pass
918 def after_export(self, queryset, data, *args, **kwargs):
919 """
920 Override to add additional logic. Does nothing by default.
921 """
922 pass
924 def export_field(self, field, obj):
925 field_name = self.get_field_name(field)
926 dehydrate_method = field.get_dehydrate_method(field_name)
928 method = getattr(self, dehydrate_method, None)
929 if method is not None:
930 return method(obj)
931 return field.export(obj)
933 def get_export_fields(self):
934 return self.get_fields()
936 def export_resource(self, obj):
937 return [self.export_field(field, obj) for field in self.get_export_fields()]
939 def get_export_headers(self):
940 headers = [
941 force_str(field.column_name) for field in self.get_export_fields()]
942 return headers
944 def get_user_visible_headers(self):
945 headers = [
946 force_str(field.column_name) for field in self.get_user_visible_fields()]
947 return headers
949 def get_user_visible_fields(self):
950 return self.get_fields()
952 def iter_queryset(self, queryset):
953 if not isinstance(queryset, QuerySet):
954 yield from queryset
955 elif queryset._prefetch_related_lookups:
956 # Django's queryset.iterator ignores prefetch_related which might result
957 # in an excessive amount of db calls. Therefore we use pagination
958 # as a work-around
959 if not queryset.query.order_by:
960 # Paginator() throws a warning if there is no sorting
961 # attached to the queryset
962 queryset = queryset.order_by('pk')
963 paginator = Paginator(queryset, self.get_chunk_size())
964 for index in range(paginator.num_pages):
965 yield from paginator.get_page(index + 1)
966 else:
967 yield from queryset.iterator(chunk_size=self.get_chunk_size())
969 def export(self, queryset=None, *args, **kwargs):
970 """
971 Exports a resource.
972 """
974 self.before_export(queryset, *args, **kwargs)
976 if queryset is None:
977 queryset = self.get_queryset()
978 headers = self.get_export_headers()
979 data = tablib.Dataset(headers=headers)
981 for obj in self.iter_queryset(queryset):
982 data.append(self.export_resource(obj))
984 self.after_export(queryset, data, *args, **kwargs)
986 return data
989class ModelDeclarativeMetaclass(DeclarativeMetaclass):
991 def __new__(cls, name, bases, attrs):
992 new_class = super().__new__(cls, name, bases, attrs)
994 opts = new_class._meta
996 if not opts.instance_loader_class: 996 ↛ 999line 996 didn't jump to line 999, because the condition on line 996 was never false
997 opts.instance_loader_class = ModelInstanceLoader
999 if opts.model:
1000 model_opts = opts.model._meta
1001 declared_fields = new_class.fields
1003 field_list = []
1004 for f in sorted(model_opts.fields + model_opts.many_to_many):
1005 if opts.fields is not None and not f.name in opts.fields:
1006 continue
1007 if opts.exclude and f.name in opts.exclude: 1007 ↛ 1008line 1007 didn't jump to line 1008, because the condition on line 1007 was never true
1008 continue
1009 if f.name in declared_fields: 1009 ↛ 1010line 1009 didn't jump to line 1010, because the condition on line 1009 was never true
1010 continue
1012 field = new_class.field_from_django_field(f.name, f,
1013 readonly=False)
1014 field_list.append((f.name, field, ))
1016 new_class.fields.update(OrderedDict(field_list))
1018 # add fields that follow relationships
1019 if opts.fields is not None:
1020 field_list = []
1021 for field_name in opts.fields:
1022 if field_name in declared_fields:
1023 continue
1024 if field_name.find('__') == -1: 1024 ↛ 1025line 1024 didn't jump to line 1025, because the condition on line 1024 was never true
1025 continue
1027 model = opts.model
1028 attrs = field_name.split('__')
1029 for i, attr in enumerate(attrs):
1030 verbose_path = ".".join([opts.model.__name__] + attrs[0:i+1])
1032 try:
1033 f = model._meta.get_field(attr)
1034 except FieldDoesNotExist as e:
1035 logger.debug(e, exc_info=e)
1036 raise FieldDoesNotExist(
1037 "%s: %s has no field named '%s'" %
1038 (verbose_path, model.__name__, attr))
1040 if i < len(attrs) - 1:
1041 # We're not at the last attribute yet, so check
1042 # that we're looking at a relation, and move on to
1043 # the next model.
1044 if isinstance(f, ForeignObjectRel): 1044 ↛ 1045line 1044 didn't jump to line 1045, because the condition on line 1044 was never true
1045 model = get_related_model(f)
1046 else:
1047 if get_related_model(f) is None: 1047 ↛ 1048line 1047 didn't jump to line 1048, because the condition on line 1047 was never true
1048 raise KeyError(
1049 '%s is not a relation' % verbose_path)
1050 model = get_related_model(f)
1052 if isinstance(f, ForeignObjectRel): 1052 ↛ 1053line 1052 didn't jump to line 1053, because the condition on line 1052 was never true
1053 f = f.field
1055 field = new_class.field_from_django_field(field_name, f,
1056 readonly=True)
1057 field_list.append((field_name, field))
1059 new_class.fields.update(OrderedDict(field_list))
1061 return new_class
1064class ModelResource(Resource, metaclass=ModelDeclarativeMetaclass):
1065 """
1066 ModelResource is Resource subclass for handling Django models.
1067 """
1069 DEFAULT_RESOURCE_FIELD = Field
1071 WIDGETS_MAP = {
1072 'ManyToManyField': 'get_m2m_widget',
1073 'OneToOneField': 'get_fk_widget',
1074 'ForeignKey': 'get_fk_widget',
1075 'CharField': widgets.CharWidget,
1076 'DecimalField': widgets.DecimalWidget,
1077 'DateTimeField': widgets.DateTimeWidget,
1078 'DateField': widgets.DateWidget,
1079 'TimeField': widgets.TimeWidget,
1080 'DurationField': widgets.DurationWidget,
1081 'FloatField': widgets.FloatWidget,
1082 'IntegerField': widgets.IntegerWidget,
1083 'PositiveIntegerField': widgets.IntegerWidget,
1084 'BigIntegerField': widgets.IntegerWidget,
1085 'PositiveSmallIntegerField': widgets.IntegerWidget,
1086 'SmallIntegerField': widgets.IntegerWidget,
1087 'SmallAutoField': widgets.IntegerWidget,
1088 'AutoField': widgets.IntegerWidget,
1089 'BigAutoField': widgets.IntegerWidget,
1090 'NullBooleanField': widgets.BooleanWidget,
1091 'BooleanField': widgets.BooleanWidget,
1092 'JSONField': widgets.JSONWidget,
1093 }
1095 @classmethod
1096 def get_m2m_widget(cls, field):
1097 """
1098 Prepare widget for m2m field
1099 """
1100 return functools.partial(
1101 widgets.ManyToManyWidget,
1102 model=get_related_model(field))
1104 @classmethod
1105 def get_fk_widget(cls, field):
1106 """
1107 Prepare widget for fk and o2o fields
1108 """
1110 model = get_related_model(field)
1112 use_natural_foreign_keys = (
1113 has_natural_foreign_key(model) and cls._meta.use_natural_foreign_keys
1114 )
1116 return functools.partial(
1117 widgets.ForeignKeyWidget,
1118 model=model,
1119 use_natural_foreign_keys=use_natural_foreign_keys)
1121 @classmethod
1122 def widget_from_django_field(cls, f, default=widgets.Widget):
1123 """
1124 Returns the widget that would likely be associated with each
1125 Django type.
1127 Includes mapping of Postgres Array field. In the case that
1128 psycopg2 is not installed, we consume the error and process the field
1129 regardless.
1130 """
1131 result = default
1132 internal_type = ""
1133 if callable(getattr(f, "get_internal_type", None)): 1133 ↛ 1136line 1133 didn't jump to line 1136, because the condition on line 1133 was never false
1134 internal_type = f.get_internal_type()
1136 if internal_type in cls.WIDGETS_MAP:
1137 result = cls.WIDGETS_MAP[internal_type]
1138 if isinstance(result, str):
1139 result = getattr(cls, result)(f)
1140 else:
1141 try:
1142 from django.contrib.postgres.fields import ArrayField
1143 except ImportError:
1144 # ImportError: No module named psycopg2.extras
1145 class ArrayField:
1146 pass
1148 if isinstance(f, ArrayField): 1148 ↛ 1149line 1148 didn't jump to line 1149, because the condition on line 1148 was never true
1149 return widgets.SimpleArrayWidget
1151 return result
1153 @classmethod
1154 def widget_kwargs_for_field(self, field_name):
1155 """
1156 Returns widget kwargs for given field_name.
1157 """
1158 if self._meta.widgets: 1158 ↛ 1159line 1158 didn't jump to line 1159, because the condition on line 1158 was never true
1159 return self._meta.widgets.get(field_name, {})
1160 return {}
1162 @classmethod
1163 def field_from_django_field(cls, field_name, django_field, readonly):
1164 """
1165 Returns a Resource Field instance for the given Django model field.
1166 """
1168 FieldWidget = cls.widget_from_django_field(django_field)
1169 widget_kwargs = cls.widget_kwargs_for_field(field_name)
1170 field = cls.DEFAULT_RESOURCE_FIELD(
1171 attribute=field_name,
1172 column_name=field_name,
1173 widget=FieldWidget(**widget_kwargs),
1174 readonly=readonly,
1175 default=django_field.default,
1176 )
1177 return field
1179 def get_queryset(self):
1180 """
1181 Returns a queryset of all objects for this model. Override this if you
1182 want to limit the returned queryset.
1183 """
1184 return self._meta.model.objects.all()
1186 def init_instance(self, row=None):
1187 """
1188 Initializes a new Django model.
1189 """
1190 return self._meta.model()
1192 def after_import(self, dataset, result, using_transactions, dry_run, **kwargs):
1193 """
1194 Reset the SQL sequences after new objects are imported
1195 """
1196 # Adapted from django's loaddata
1197 if not dry_run and any(r.import_type == RowResult.IMPORT_TYPE_NEW for r in result.rows):
1198 db_connection = self.get_db_connection_name()
1199 connection = connections[db_connection]
1200 sequence_sql = connection.ops.sequence_reset_sql(no_style(), [self._meta.model])
1201 if sequence_sql:
1202 cursor = connection.cursor()
1203 try:
1204 for line in sequence_sql:
1205 cursor.execute(line)
1206 finally:
1207 cursor.close()
1209 @classmethod
1210 def get_display_name(cls):
1211 if hasattr(cls._meta, 'name'):
1212 return cls._meta.name
1213 return cls.__name__
1216def modelresource_factory(model, resource_class=ModelResource):
1217 """
1218 Factory for creating ``ModelResource`` class for given Django model.
1219 """
1220 attrs = {'model': model}
1221 Meta = type(str('Meta'), (object,), attrs)
1223 class_name = model.__name__ + str('Resource')
1225 class_attrs = {
1226 'Meta': Meta,
1227 }
1229 metaclass = ModelDeclarativeMetaclass
1230 return metaclass(class_name, (resource_class,), class_attrs)